From 2cb92404b153897440c4906a5548dcfb4889a9ad Mon Sep 17 00:00:00 2001 From: Jeongkyu Shin Date: Sun, 11 Oct 2026 12:09:46 +0900 Subject: [PATCH 1/7] fix: clean up paged compatibility and SSM precision Remove the unused dense and rotating paged compatibility paths while preserving pooled decode coverage with an explicit float32 dense oracle. The oracle retains its absolute RMS checks and adds normalized RMS checks so a one-percent output perturbation is rejected. Promote both dt operands before the compiled SSM softplus so bf16 activation parameters cannot round the float32 recurrence. Add the paired/null serving benchmark driver and analyzer for the pending Gemma 3 and Llama 4 storage decision. Refs #2230 --- .../analyze.py | 140 ++++++ .../run.sh | 130 ++++++ src/lib/mlxcel-core/cpp/mlx_cxx_bridge.cpp | 252 ---------- src/lib/mlxcel-core/cpp/mlx_cxx_bridge.h | 23 - src/lib/mlxcel-core/cpp/mlx_cxx_kernels.cpp | 6 +- src/lib/mlxcel-core/src/cache.rs | 162 ------- src/lib/mlxcel-core/src/cache/attend.rs | 11 +- src/lib/mlxcel-core/src/ffi_tests.rs | 350 ++++++-------- src/lib/mlxcel-core/src/layers.rs | 435 +----------------- src/lib/mlxcel-core/src/lib.rs | 30 -- 10 files changed, 427 insertions(+), 1112 deletions(-) create mode 100755 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/analyze.py create mode 100755 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/run.sh diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/analyze.py b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/analyze.py new file mode 100755 index 000000000..813f5f352 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/analyze.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Summarize the model-owned decode-storage paired/null serving sweep.""" + +from __future__ import annotations + +import re +import statistics +from dataclasses import dataclass +from pathlib import Path + + +DATA = Path(__file__).resolve().parent +LOG_NAME = re.compile( + r"client-(?Pgemma3|llama4)-p(?P512|4096)-" + r"r(?P[1-5])-(?Ppaged-a|dense|paged-b)\.txt" +) +TABLE_ROW = re.compile( + r"^\s*(?P1|4|8)\s+\d+\s+(?P\d+)\s+" + r"(?:\d+(?:\.\d+)?\s+){3}(?P\d+(?:\.\d+)?)\s*$" +) + + +@dataclass(frozen=True) +class Run: + """Aggregate throughput from one fresh-server arm.""" + + model: str + prompt_tokens: int + round: int + arm: str + concurrency: int + aggregate_tok_s: float + + +def percent_delta(value: float, baseline: float) -> float: + """Return the percentage delta of ``value`` from ``baseline``.""" + + return 100.0 * (value / baseline - 1.0) + + +def read_runs() -> list[Run]: + """Read all expected client logs and reject incomplete or failed arms.""" + + runs: list[Run] = [] + for path in sorted(DATA.glob("client-*.txt")): + match = LOG_NAME.fullmatch(path.name) + if match is None: + continue + rows = [] + for line in path.read_text().splitlines(): + if row := TABLE_ROW.match(line): + if int(row["failed"]) != 0: + raise ValueError(f"{path}: concurrency {row['concurrency']} has failures") + rows.append( + Run( + model=match["model"], + prompt_tokens=int(match["prompt"]), + round=int(match["round"]), + arm=match["arm"], + concurrency=int(row["concurrency"]), + aggregate_tok_s=float(row["aggregate"]), + ) + ) + if len(rows) != 3: + raise ValueError(f"{path}: expected three concurrency rows, found {len(rows)}") + runs.extend(rows) + + expected = 2 * 2 * 5 * 3 * 3 + if len(runs) != expected: + raise ValueError(f"expected {expected} measurements, found {len(runs)}") + return runs + + +def main() -> int: + """Print per-cell medians, paired deltas, null spreads, and family decisions.""" + + by_key = { + (run.model, run.prompt_tokens, run.concurrency, run.round, run.arm): run.aggregate_tok_s + for run in read_runs() + } + decisions: dict[str, tuple[bool, bool]] = {} + + print("| Model | Prompt | C | paged-a median | dense median | paged-b median | dense paired delta | null spread | Cell |") + print("|---|---:|---:|---:|---:|---:|---:|---:|---|") + for model in ("gemma3", "llama4"): + any_dense_win = False + any_paged_win = False + for prompt in (512, 4096): + for concurrency in (1, 4, 8): + arm_values = { + arm: [ + by_key[(model, prompt, concurrency, round_no, arm)] + for round_no in range(1, 6) + ] + for arm in ("paged-a", "dense", "paged-b") + } + paired = [] + null = [] + for index in range(5): + paired.append( + percent_delta( + arm_values["dense"][index], arm_values["paged-a"][index] + ) + ) + null.append( + percent_delta( + arm_values["paged-b"][index], arm_values["paged-a"][index] + ) + ) + + paired_median = statistics.median(paired) + null_low, null_high = min(null), max(null) + dense_win = paired_median > max(0.0, null_high) + paged_win = paired_median < min(0.0, null_low) + any_dense_win |= dense_win + any_paged_win |= paged_win + verdict = "dense" if dense_win else "paged" if paged_win else "unresolved" + medians = { + arm: statistics.median(values) for arm, values in arm_values.items() + } + print( + f"| {model} | {prompt} | {concurrency} | {medians['paged-a']:.1f} | " + f"{medians['dense']:.1f} | {medians['paged-b']:.1f} | " + f"{paired_median:+.2f}% ({min(paired):+.2f}..{max(paired):+.2f}) | " + f"{null_low:+.2f}..{null_high:+.2f}% | {verdict} |" + ) + decisions[model] = (any_dense_win, any_paged_win) + + print() + for model, (any_dense_win, any_paged_win) in decisions.items(): + use_dense = any_dense_win and not any_paged_win + print( + f"{model}: supports_paged_decode_backend={'false' if use_dense else 'true'} " + f"(dense_clear_cell={any_dense_win}, paged_clear_cell={any_paged_win})" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/run.sh b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/run.sh new file mode 100755 index 000000000..3654f5b67 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/run.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +WORKTREE=/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230 +BIN=/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/target/release/mlxcel +CLIENT=/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/scripts/bench_serving_concurrency.py +MODEL_ROOT=/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/models/mlx +DATA=/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11 +PORT=18080 + +export MLXCEL_CCCL_DIR=/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/target/release/build/mlxcel-core-4c8821fbbcc3d8f4/out/build/include/cccl +export MLXCEL_CUTLASS_DIR=/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/target/release/build/mlxcel-core-4c8821fbbcc3d8f4/out/build/include + +SERVER_PID= +cleanup_server() { + if [[ -n "${SERVER_PID}" ]] && kill -0 "${SERVER_PID}" 2>/dev/null; then + kill "${SERVER_PID}" 2>/dev/null || true + wait "${SERVER_PID}" 2>/dev/null || true + fi + SERVER_PID= +} +trap cleanup_server EXIT INT TERM + +mkdir -p "${DATA}" +exec > >(tee -a "${DATA}/run.log") 2>&1 + +for required in "${BIN}" "${CLIENT}" "${MODEL_ROOT}/gemma-3-1b-it-4bit" "${MODEL_ROOT}/llama-4-scout-17b-16e-instruct-4bit" "${MLXCEL_CCCL_DIR}" "${MLXCEL_CUTLASS_DIR}"; do + if [[ ! -e "${required}" ]]; then + echo "missing required benchmark path: ${required}" >&2 + exit 2 + fi +done + +if [[ -e "${DATA}/complete" ]]; then + echo "benchmark already completed; remove ${DATA}/complete to rerun" >&2 + exit 2 +fi + +{ + echo "started_utc=$(date -u +%FT%TZ)" + echo "worktree=${WORKTREE}" + echo "binary=${BIN}" + echo "binary_sha256=$(sha256sum "${BIN}" | awk '{print $1}')" + echo "commit=$(git -C "${WORKTREE}" rev-parse HEAD)" + echo "mlx_pin=$(git -C "${WORKTREE}/src/lib/mlx-cpp" rev-parse HEAD)" + echo "host=$(hostname)" + uname -a + nvidia-smi --query-gpu=name,driver_version,pci.bus_id --format=csv,noheader +} > "${DATA}/environment.txt" + +: > "${DATA}/schedule.tsv" +printf 'model\tprompt_tokens\tround\tposition\tarm\tbackend\tstarted_utc\n' >> "${DATA}/schedule.tsv" + +wait_for_server() { + local attempt + for attempt in $(seq 1 600); do + if curl --fail --silent "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1 || curl --fail --silent "http://127.0.0.1:${PORT}/v1/models" >/dev/null 2>&1; then + return 0 + fi + if ! kill -0 "${SERVER_PID}" 2>/dev/null; then + return 1 + fi + sleep 1 + done + return 1 +} + +run_arm() { + local model_name=$1 + local model_path=$2 + local prompt_tokens=$3 + local round=$4 + local position=$5 + local arm=$6 + local backend=paged + if [[ "${arm}" == dense ]]; then + backend=dense + fi + + local stem="${model_name}-p${prompt_tokens}-r${round}-${arm}" + local server_log="${DATA}/server-${stem}.log" + local client_log="${DATA}/client-${stem}.txt" + + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "${model_name}" "${prompt_tokens}" "${round}" "${position}" "${arm}" "${backend}" "$(date -u +%FT%TZ)" >> "${DATA}/schedule.tsv" + echo "START ${stem} $(date -u +%FT%TZ)" + "${BIN}" serve -m "${model_path}" --port "${PORT}" --metrics --parallel 8 --decode-storage-backend "${backend}" > "${server_log}" 2>&1 & + SERVER_PID=$! + + if ! wait_for_server; then + echo "server failed to become ready for ${stem}" >&2 + tail -n 100 "${server_log}" >&2 || true + exit 3 + fi + + python3 "${CLIENT}" --port "${PORT}" --concurrency 1,4,8 --prompt-tokens "${prompt_tokens}" --max-tokens 128 --metrics > "${client_log}" 2>&1 + if ! awk '/^[[:space:]]*[148][[:space:]]/ { rows += 1; if ($3 != 0) failed = 1 } END { exit !(rows == 3 && failed == 0) }' "${client_log}"; then + echo "client run reported missing rows or request failures for ${stem}" >&2 + cat "${client_log}" >&2 + exit 4 + fi + + cleanup_server + sleep 5 + echo "DONE ${stem} $(date -u +%FT%TZ)" +} + +for model_name in gemma3 llama4; do + case "${model_name}" in + gemma3) model_path="${MODEL_ROOT}/gemma-3-1b-it-4bit" ;; + llama4) model_path="${MODEL_ROOT}/llama-4-scout-17b-16e-instruct-4bit" ;; + esac + for prompt_tokens in 512 4096; do + for round in 1 2 3 4 5; do + case $(( (round - 1) % 3 )) in + 0) order=(paged-a dense paged-b) ;; + 1) order=(dense paged-b paged-a) ;; + 2) order=(paged-b paged-a dense) ;; + esac + position=0 + for arm in "${order[@]}"; do + position=$((position + 1)) + run_arm "${model_name}" "${model_path}" "${prompt_tokens}" "${round}" "${position}" "${arm}" + done + done + done +done + +echo "completed_utc=$(date -u +%FT%TZ)" >> "${DATA}/environment.txt" +touch "${DATA}/complete" +echo "ALLDONE $(date -u +%FT%TZ)" diff --git a/src/lib/mlxcel-core/cpp/mlx_cxx_bridge.cpp b/src/lib/mlxcel-core/cpp/mlx_cxx_bridge.cpp index aa7686d6a..70cf07778 100644 --- a/src/lib/mlxcel-core/cpp/mlx_cxx_bridge.cpp +++ b/src/lib/mlxcel-core/cpp/mlx_cxx_bridge.cpp @@ -4109,258 +4109,6 @@ std::unique_ptr fast_scaled_dot_product_attention_causal( )); } -std::unique_ptr paged_decode_attention_dense_compat( - const MlxArray& q, - rust::Slice cache_keys, - rust::Slice cache_values, - rust::Slice kv_lens, - rust::Slice block_tables, - rust::Slice block_table_offsets, - int32_t block_size, - float scale -) { - if (block_size <= 0) { - throw std::invalid_argument("paged_decode_attention_dense_compat: block_size must be > 0"); - } - - const auto q_shape = q.inner.shape(); - if (q_shape.size() != 4 || q_shape[2] != 1) { - throw std::invalid_argument("paged_decode_attention_dense_compat: expected q shape [B, H, 1, D]"); - } - - const size_t batch = static_cast(q_shape[0]); - if (cache_keys.size() != batch || cache_values.size() != batch || kv_lens.size() != batch) { - throw std::invalid_argument("paged_decode_attention_dense_compat: batch metadata length mismatch"); - } - if (block_table_offsets.size() != batch + 1) { - throw std::invalid_argument("paged_decode_attention_dense_compat: block_table_offsets must have length B + 1"); - } - - std::vector outputs; - outputs.reserve(batch); - - for (size_t batch_idx = 0; batch_idx < batch; ++batch_idx) { - const MlxArray* key_cache = cache_keys[batch_idx]; - const MlxArray* value_cache = cache_values[batch_idx]; - if (key_cache == nullptr || value_cache == nullptr) { - throw std::invalid_argument("paged_decode_attention_dense_compat: null cache pointer"); - } - - const int32_t kv_len = kv_lens[batch_idx]; - if (kv_len <= 0) { - throw std::invalid_argument("paged_decode_attention_dense_compat: kv_len must be > 0"); - } - - const int32_t table_begin = block_table_offsets[batch_idx]; - const int32_t table_end = block_table_offsets[batch_idx + 1]; - if (table_begin < 0 || table_end < table_begin || - static_cast(table_end) > block_tables.size()) { - throw std::invalid_argument("paged_decode_attention_dense_compat: invalid block table offsets"); - } - - std::vector key_blocks; - std::vector value_blocks; - key_blocks.reserve(table_end - table_begin); - value_blocks.reserve(table_end - table_begin); - - for (int32_t table_idx = table_begin; table_idx < table_end; ++table_idx) { - const int32_t logical_block = block_tables[table_idx]; - if (logical_block < 0) { - throw std::invalid_argument("paged_decode_attention_dense_compat: block indices must be non-negative"); - } - - const int32_t block_start = logical_block * block_size; - if (block_start >= kv_len) { - continue; - } - const int32_t block_end = std::min(block_start + block_size, kv_len); - - key_blocks.push_back(mlx::core::slice( - key_cache->inner, - {0, 0, block_start, 0}, - {1, static_cast(key_cache->inner.shape(1)), block_end, - static_cast(key_cache->inner.shape(3))} - )); - value_blocks.push_back(mlx::core::slice( - value_cache->inner, - {0, 0, block_start, 0}, - {1, static_cast(value_cache->inner.shape(1)), block_end, - static_cast(value_cache->inner.shape(3))} - )); - } - - if (key_blocks.empty() || value_blocks.empty()) { - throw std::invalid_argument("paged_decode_attention_dense_compat: empty visible block list"); - } - - array key_visible = - key_blocks.size() == 1 ? key_blocks.front() : mlx::core::concatenate(key_blocks, 2); - array value_visible = value_blocks.size() == 1 - ? value_blocks.front() - : mlx::core::concatenate(value_blocks, 2); - - auto q_i = mlx::core::slice( - q.inner, - {static_cast(batch_idx), 0, 0, 0}, - {static_cast(batch_idx + 1), static_cast(q_shape[1]), 1, - static_cast(q_shape[3])} - ); - auto attn_i = mlx::core::fast::scaled_dot_product_attention( - q_i, key_visible, value_visible, scale, "", std::nullopt - ); - outputs.push_back(std::move(attn_i)); - } - - if (outputs.empty()) { - throw std::invalid_argument("paged_decode_attention_dense_compat: empty batch"); - } - if (outputs.size() == 1) { - return std::make_unique(std::move(outputs.front())); - } - return std::make_unique(mlx::core::concatenate(outputs, 0)); -} - -std::unique_ptr paged_decode_attention_rotating_compat( - const MlxArray& q, - rust::Slice cache_keys, - rust::Slice cache_values, - rust::Slice kv_lens, - rust::Slice logical_starts, - int32_t block_size, - float scale -) { - if (block_size <= 0) { - throw std::invalid_argument("paged_decode_attention_rotating_compat: block_size must be > 0"); - } - - const auto q_shape = q.inner.shape(); - if (q_shape.size() != 4 || q_shape[2] != 1) { - throw std::invalid_argument("paged_decode_attention_rotating_compat: expected q shape [B, H, 1, D]"); - } - - const size_t batch = static_cast(q_shape[0]); - if (cache_keys.size() != batch || cache_values.size() != batch || - kv_lens.size() != batch || logical_starts.size() != batch) { - throw std::invalid_argument("paged_decode_attention_rotating_compat: batch metadata length mismatch"); - } - - std::vector outputs; - outputs.reserve(batch); - - for (size_t batch_idx = 0; batch_idx < batch; ++batch_idx) { - const MlxArray* key_cache = cache_keys[batch_idx]; - const MlxArray* value_cache = cache_values[batch_idx]; - if (key_cache == nullptr || value_cache == nullptr) { - throw std::invalid_argument("paged_decode_attention_rotating_compat: null cache pointer"); - } - - const auto key_shape = key_cache->inner.shape(); - const auto value_shape = value_cache->inner.shape(); - if (key_shape.size() != 4 || value_shape.size() != 4) { - throw std::invalid_argument("paged_decode_attention_rotating_compat: cache tensors must have rank 4"); - } - - const int32_t kv_len = kv_lens[batch_idx]; - const int32_t logical_start = logical_starts[batch_idx]; - const int32_t buffer_len = static_cast(key_shape[2]); - if (kv_len <= 0) { - throw std::invalid_argument("paged_decode_attention_rotating_compat: kv_len must be > 0"); - } - if (buffer_len <= 0) { - throw std::invalid_argument("paged_decode_attention_rotating_compat: buffer_len must be > 0"); - } - if (logical_start < 0 || logical_start >= buffer_len) { - throw std::invalid_argument("paged_decode_attention_rotating_compat: logical_start out of range"); - } - - std::vector key_blocks; - std::vector value_blocks; - const int32_t block_count = (kv_len + block_size - 1) / block_size; - key_blocks.reserve(block_count); - value_blocks.reserve(block_count); - - for (int32_t logical_block = 0; logical_block < block_count; ++logical_block) { - const int32_t logical_pos = logical_block * block_size; - const int32_t logical_end = std::min(logical_pos + block_size, kv_len); - const int32_t token_count = logical_end - logical_pos; - if (token_count <= 0) { - continue; - } - - const int32_t physical_start = (logical_start + logical_pos) % buffer_len; - const int32_t physical_end = physical_start + token_count; - if (physical_end <= buffer_len) { - key_blocks.push_back(mlx::core::slice( - key_cache->inner, - {0, 0, physical_start, 0}, - {1, static_cast(key_shape[1]), physical_end, static_cast(key_shape[3])} - )); - value_blocks.push_back(mlx::core::slice( - value_cache->inner, - {0, 0, physical_start, 0}, - {1, static_cast(value_shape[1]), physical_end, static_cast(value_shape[3])} - )); - } else { - const int32_t tail_count = buffer_len - physical_start; - const int32_t head_count = token_count - tail_count; - - auto key_tail = mlx::core::slice( - key_cache->inner, - {0, 0, physical_start, 0}, - {1, static_cast(key_shape[1]), buffer_len, static_cast(key_shape[3])} - ); - auto key_head = mlx::core::slice( - key_cache->inner, - {0, 0, 0, 0}, - {1, static_cast(key_shape[1]), head_count, static_cast(key_shape[3])} - ); - key_blocks.push_back(mlx::core::concatenate(std::vector{key_tail, key_head}, 2)); - - auto value_tail = mlx::core::slice( - value_cache->inner, - {0, 0, physical_start, 0}, - {1, static_cast(value_shape[1]), buffer_len, static_cast(value_shape[3])} - ); - auto value_head = mlx::core::slice( - value_cache->inner, - {0, 0, 0, 0}, - {1, static_cast(value_shape[1]), head_count, static_cast(value_shape[3])} - ); - value_blocks.push_back(mlx::core::concatenate(std::vector{value_tail, value_head}, 2)); - } - } - - if (key_blocks.empty() || value_blocks.empty()) { - throw std::invalid_argument("paged_decode_attention_rotating_compat: empty visible block list"); - } - - array key_visible = - key_blocks.size() == 1 ? key_blocks.front() : mlx::core::concatenate(key_blocks, 2); - array value_visible = value_blocks.size() == 1 - ? value_blocks.front() - : mlx::core::concatenate(value_blocks, 2); - - auto q_i = mlx::core::slice( - q.inner, - {static_cast(batch_idx), 0, 0, 0}, - {static_cast(batch_idx + 1), static_cast(q_shape[1]), 1, - static_cast(q_shape[3])} - ); - auto attn_i = mlx::core::fast::scaled_dot_product_attention( - q_i, key_visible, value_visible, scale, "", std::nullopt - ); - outputs.push_back(std::move(attn_i)); - } - - if (outputs.empty()) { - throw std::invalid_argument("paged_decode_attention_rotating_compat: empty batch"); - } - if (outputs.size() == 1) { - return std::make_unique(std::move(outputs.front())); - } - return std::make_unique(mlx::core::concatenate(outputs, 0)); -} - namespace { bool sdpa_supports_fast_path_impl( const array& q, diff --git a/src/lib/mlxcel-core/cpp/mlx_cxx_bridge.h b/src/lib/mlxcel-core/cpp/mlx_cxx_bridge.h index ab2bd7a9c..c5f364511 100644 --- a/src/lib/mlxcel-core/cpp/mlx_cxx_bridge.h +++ b/src/lib/mlxcel-core/cpp/mlx_cxx_bridge.h @@ -1126,29 +1126,6 @@ std::unique_ptr fast_scaled_dot_product_attention_causal( float scale ); -// Decode-only paged attention over dense compatibility KV caches. -std::unique_ptr paged_decode_attention_dense_compat( - const MlxArray& q, - rust::Slice cache_keys, - rust::Slice cache_values, - rust::Slice kv_lens, - rust::Slice block_tables, - rust::Slice block_table_offsets, - int32_t block_size, - float scale -); - -// Decode-only paged attention over rotating ring-buffer KV caches. -std::unique_ptr paged_decode_attention_rotating_compat( - const MlxArray& q, - rust::Slice cache_keys, - rust::Slice cache_values, - rust::Slice kv_lens, - rust::Slice logical_starts, - int32_t block_size, - float scale -); - // Upstream MLX SDPA capability helpers for Metal/NAX instrumentation. bool sdpa_supports_fast_path( const MlxArray& q, diff --git a/src/lib/mlxcel-core/cpp/mlx_cxx_kernels.cpp b/src/lib/mlxcel-core/cpp/mlx_cxx_kernels.cpp index 2b44cc1d9..cbb0848d8 100644 --- a/src/lib/mlxcel-core/cpp/mlx_cxx_kernels.cpp +++ b/src/lib/mlxcel-core/cpp/mlx_cxx_kernels.cpp @@ -858,7 +858,11 @@ const mlxcel::KernelPorts& ssm_ports() { get_compiled_compute_dt() { auto fn = [](const std::vector& inputs) -> std::vector { auto dt = mlx::core::astype(inputs[0], mlx::core::float32); - const auto& dt_bias = inputs[1]; + // MLX's binary type promotion preserves the bf16 bias dtype here, + // which would round the softplus result before the float32 state + // update. Promote both operands explicitly so the compiled path + // matches the model graph and the kernel's float32 recurrence. + auto dt_bias = mlx::core::astype(inputs[1], mlx::core::float32); const auto& lo = inputs[2]; const auto& hi = inputs[3]; auto result = mlx::core::add(dt, dt_bias); diff --git a/src/lib/mlxcel-core/src/cache.rs b/src/lib/mlxcel-core/src/cache.rs index fa93e1892..94b23ee34 100644 --- a/src/lib/mlxcel-core/src/cache.rs +++ b/src/lib/mlxcel-core/src/cache.rs @@ -6333,139 +6333,6 @@ impl BatchedAttentionMetadata { } } -/// Decode-only paged attention metadata derived from per-sequence KV lengths. -/// -/// The current dense-compat kernel treats `block_tables` as logical block -/// indices (`0..num_blocks`) for each sequence. A future physical paged-KV -/// backend can reuse the same shape while replacing these entries with actual -/// physical block identifiers. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct PagedDecodeMetadata { - pub block_size: i32, - pub kv_lens: Vec, - pub block_table_offsets: Vec, - pub block_tables: Vec, -} - -/// Decode-only paged attention metadata for ring-buffer-backed rotating caches. -/// -/// `logical_starts[i]` identifies the physical buffer index of the oldest -/// visible token for sequence `i`, allowing native paged decode kernels to -/// gather wrapped sliding-window buffers without first materializing a dense -/// linearized copy in Rust. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct RotatingPagedDecodeMetadata { - pub block_size: i32, - pub kv_lens: Vec, - pub logical_starts: Vec, -} - -impl RotatingPagedDecodeMetadata { - pub fn from_parts( - kv_lens: &[i32], - logical_starts: &[i32], - block_size: i32, - ) -> Result { - if block_size <= 0 { - return Err(format!( - "rotating paged decode metadata requires block_size > 0, got {block_size}" - )); - } - if kv_lens.len() != logical_starts.len() { - return Err(format!( - "rotating paged decode metadata length mismatch: {} kv_lens vs {} logical_starts", - kv_lens.len(), - logical_starts.len() - )); - } - for (&kv_len, &logical_start) in kv_lens.iter().zip(logical_starts.iter()) { - if kv_len < 0 { - return Err(format!( - "rotating paged decode metadata requires non-negative kv lengths, got {kv_len}" - )); - } - if logical_start < 0 { - return Err(format!( - "rotating paged decode metadata requires non-negative logical starts, got {logical_start}" - )); - } - if kv_len > 0 && logical_start >= kv_len { - return Err(format!( - "rotating paged decode metadata requires logical_start < kv_len when kv_len > 0, got logical_start={logical_start}, kv_len={kv_len}" - )); - } - } - - Ok(Self { - block_size, - kv_lens: kv_lens.to_vec(), - logical_starts: logical_starts.to_vec(), - }) - } - - pub fn len(&self) -> usize { - self.kv_lens.len() - } - - pub fn is_empty(&self) -> bool { - self.kv_lens.is_empty() - } -} - -impl PagedDecodeMetadata { - pub fn from_attention_metadata( - metadata: &BatchedAttentionMetadata, - block_size: i32, - ) -> Result { - Self::from_visible_lengths(&metadata.kv_lens, block_size) - } - - pub fn from_visible_lengths(kv_lens: &[i32], block_size: i32) -> Result { - if block_size <= 0 { - return Err(format!( - "paged decode metadata requires block_size > 0, got {block_size}" - )); - } - - let mut block_table_offsets = Vec::with_capacity(kv_lens.len() + 1); - let mut block_tables = Vec::new(); - block_table_offsets.push(0); - - for &kv_len in kv_lens { - if kv_len < 0 { - return Err(format!( - "paged decode metadata requires non-negative kv lengths, got {kv_len}" - )); - } - - let block_count = if kv_len == 0 { - 0 - } else { - (kv_len + block_size - 1) / block_size - }; - for logical_block in 0..block_count { - block_tables.push(logical_block); - } - block_table_offsets.push(block_tables.len() as i32); - } - - Ok(Self { - block_size, - kv_lens: kv_lens.to_vec(), - block_table_offsets, - block_tables, - }) - } - - pub fn len(&self) -> usize { - self.kv_lens.len() - } - - pub fn is_empty(&self) -> bool { - self.kv_lens.is_empty() - } -} - impl Default for ChunkedKVCache { fn default() -> Self { Self::new(8192) @@ -10005,35 +9872,6 @@ mod tests { assert!(BatchedAttentionMetadata::from_kv_caches(&caches, &[1], &[0, 1]).is_err()); } - #[test] - fn paged_decode_metadata_builds_logical_block_tables() { - let metadata = BatchedAttentionMetadata { - rope_offsets: vec![0, 4], - query_lens: vec![1, 1], - kv_lens: vec![3, 5], - window_sizes: vec![0, 0], - }; - - let paged = PagedDecodeMetadata::from_attention_metadata(&metadata, 2).unwrap(); - - assert_eq!(paged.block_size, 2); - assert_eq!(paged.kv_lens, vec![3, 5]); - assert_eq!(paged.block_table_offsets, vec![0, 2, 5]); - assert_eq!(paged.block_tables, vec![0, 1, 0, 1, 2]); - } - - #[test] - fn paged_decode_metadata_rejects_invalid_block_size() { - let metadata = BatchedAttentionMetadata { - rope_offsets: vec![0], - query_lens: vec![1], - kv_lens: vec![1], - window_sizes: vec![0], - }; - - assert!(PagedDecodeMetadata::from_attention_metadata(&metadata, 0).is_err()); - } - // ----------------------------------------------------------------- // regression coverage — TurboQuant continuous-batching // batch-grow / batch-shrink mid-decode. diff --git a/src/lib/mlxcel-core/src/cache/attend.rs b/src/lib/mlxcel-core/src/cache/attend.rs index a5af55fc0..e1995b12e 100644 --- a/src/lib/mlxcel-core/src/cache/attend.rs +++ b/src/lib/mlxcel-core/src/cache/attend.rs @@ -50,12 +50,11 @@ //! dense entries for those storages, [`KvAttention`] is the trait the model //! enums implement by matching to the inner cache, and [`attend_batched_rows`] //! is the per-row batched loop every non-pooled batch takes. The -//! `DecodeBatchContext` those families used to read is gone: the -//! dense-pointer "paged compat" kernels it selected -//! (`paged_decode_attention_dense_compat`, `_rotating_compat`) were per-row -//! loops of block slices, a concat and one SDPA call per row, so a batch now -//! runs the same attention per row through its own cache without the concat -//! copy, the change ADR 0008 already made for Qwen3 and Llama 3. +//! `DecodeBatchContext` those families used to read is gone: its old +//! dense-pointer route ran per-row loops of block slices, a concat and one SDPA +//! call per row, so a batch now runs the same attention per row through its own +//! cache without the concat copy, the change ADR 0008 already made for Qwen3 +//! and Llama 3. use cxx::UniquePtr; diff --git a/src/lib/mlxcel-core/src/ffi_tests.rs b/src/lib/mlxcel-core/src/ffi_tests.rs index 233c4b2fe..0256bcbba 100644 --- a/src/lib/mlxcel-core/src/ffi_tests.rs +++ b/src/lib/mlxcel-core/src/ffi_tests.rs @@ -343,118 +343,69 @@ fn test_fast_rope_batched_non_contiguous_t1_symmetric() { ); } -#[test] -fn test_paged_decode_attention_dense_compat_matches_fallback() { - let q = from_slice_f32(&[1.0, 0.5, 0.25, 1.25], &[2, 1, 1, 2]); - - let cache_k_0 = from_slice_f32(&[1.0, 0.0, 0.0, 1.0, 1.0, 1.0], &[1, 1, 3, 2]); - let cache_v_0 = from_slice_f32(&[0.5, 0.0, 0.0, 0.5, 1.0, 1.0], &[1, 1, 3, 2]); - let cache_k_1 = from_slice_f32( - &[1.0, 0.0, 0.0, 1.0, 0.5, 0.5, 1.0, 1.0, 0.25, 1.25], - &[1, 1, 5, 2], - ); - let cache_v_1 = from_slice_f32( - &[0.2, 0.8, 0.8, 0.2, 0.5, 0.5, 1.2, 0.4, 0.3, 1.3], - &[1, 1, 5, 2], - ); - - let cache_keys = vec![ - cache_k_0.as_ref().unwrap() as *const MlxArray, - cache_k_1.as_ref().unwrap() as *const MlxArray, - ]; - let cache_values = vec![ - cache_v_0.as_ref().unwrap() as *const MlxArray, - cache_v_1.as_ref().unwrap() as *const MlxArray, - ]; - let metadata = crate::cache::PagedDecodeMetadata { - block_size: 2, - kv_lens: vec![3, 5], - block_table_offsets: vec![0, 2, 5], - block_tables: vec![0, 1, 0, 1, 2], - }; - - let fallback = crate::layers::paged_decode_attention_dense_fallback( - &q, - &cache_keys, - &cache_values, - &metadata, - 1.0, - ) - .unwrap(); - let native = crate::layers::paged_decode_attention_dense_compat( - &q, - &cache_keys, - &cache_values, - &metadata, - 1.0, - ) - .unwrap(); - - eval(&fallback); - eval(&native); - let close = allclose(&fallback, &native, 1e-5, 1e-5); - eval(&close); - assert!(item_bool(&close)); -} - -#[test] -fn test_paged_decode_attention_rotating_compat_matches_fallback_after_wrap() { - let q = from_slice_f32(&[0.5, 1.0, 1.5, 2.0], &[2, 1, 1, 2]); - - // Sequence 0 ring buffer physically stores logical tokens [2, 3, 4] as [4, 2, 3]. - let cache_k_0 = from_slice_f32(&[4.0, 0.0, 2.0, 0.0, 3.0, 0.0], &[1, 1, 3, 2]); - let cache_v_0 = from_slice_f32(&[0.4, 0.0, 0.2, 0.0, 0.3, 0.0], &[1, 1, 3, 2]); - // Sequence 1 ring buffer physically stores logical tokens [8, 9, 10, 11] as [10, 11, 8, 9]. - let cache_k_1 = from_slice_f32(&[10.0, 1.0, 11.0, 1.1, 8.0, 0.8, 9.0, 0.9], &[1, 1, 4, 2]); - let cache_v_1 = from_slice_f32(&[1.0, 0.1, 1.1, 0.11, 0.8, 0.08, 0.9, 0.09], &[1, 1, 4, 2]); - - let cache_keys = vec![ - cache_k_0.as_ref().unwrap() as *const MlxArray, - cache_k_1.as_ref().unwrap() as *const MlxArray, - ]; - let cache_values = vec![ - cache_v_0.as_ref().unwrap() as *const MlxArray, - cache_v_1.as_ref().unwrap() as *const MlxArray, - ]; - let metadata = - crate::cache::RotatingPagedDecodeMetadata::from_parts(&[3, 4], &[1, 2], 2).unwrap(); - - let fallback = crate::layers::paged_decode_attention_rotating_fallback( - &q, - &cache_keys, - &cache_values, - &metadata, - 1.0, - ) - .unwrap(); - let native = crate::layers::paged_decode_attention_rotating_compat( - &q, - &cache_keys, - &cache_values, - &metadata, - 1.0, - ) - .unwrap(); - - eval(&fallback); - eval(&native); - let close = allclose(&fallback, &native, 1e-5, 1e-5); - eval(&close); - assert!(item_bool(&close)); -} - // --------------------------------------------------------------------------- -// Pooled paged decode (#119): parity vs the dense fallback baseline. +// Pooled paged decode (#119): parity vs a dense attention reference. // // `paged_decode_attention_pooled_fallback` gathers each sequence's visible K/V // from a `PagedBlockPool` (real, possibly fragmented physical block tables), -// whereas `paged_decode_attention_dense_fallback` slices contiguous dense -// buffers with identity metadata. Both feed the same fused SDPA, so for -// identical K/V/q values the two outputs must match tightly. FP32 K/V/q is used -// so the gather (pure take/reshape/slice/transpose) and the dense slice/concat -// stay bit-exact and the only residual delta is SDPA float ordering. +// while the test-only oracle below evaluates softmax(q k^T * scale) v over the +// contiguous K/V tensors the tests build in parallel. FP32 K/V/q keeps the +// comparison focused on the pooled gather and attention computation. // --------------------------------------------------------------------------- +fn dense_reference_decode_attention( + q: &MlxArray, + keys: &[&MlxArray], + values: &[&MlxArray], + scale: f32, +) -> UniquePtr { + let q_shape = array_shape(q); + assert_eq!(q_shape.len(), 4, "reference q must have rank 4"); + assert_eq!(q_shape[2], 1, "reference q must be a decode token"); + assert_eq!(keys.len(), values.len(), "reference K/V batch mismatch"); + assert_eq!( + q_shape[0] as usize, + keys.len(), + "reference q/K batch mismatch" + ); + + let q_heads = q_shape[1]; + let mut rows = Vec::with_capacity(keys.len()); + for (batch_idx, (&key, &value)) in keys.iter().zip(values).enumerate() { + let key_shape = array_shape(key); + assert_eq!(key_shape.len(), 4, "reference K must have rank 4"); + assert_eq!( + array_shape(value), + key_shape, + "reference K/V shape mismatch" + ); + let kv_heads = key_shape[1]; + assert_eq!(q_heads % kv_heads, 0, "query heads must divide by KV heads"); + + let row = batch_idx as i32; + let q_row = slice(q, &[row, 0, 0, 0], &[row + 1, q_heads, 1, q_shape[3]]); + let q_f32 = astype(&q_row, dtype::FLOAT32); + let mut key_f32 = astype(key, dtype::FLOAT32); + let mut value_f32 = astype(value, dtype::FLOAT32); + if q_heads != kv_heads { + let repeats = q_heads / kv_heads; + key_f32 = repeat(&key_f32, repeats, 1); + value_f32 = repeat(&value_f32, repeats, 1); + } + + let key_t = transpose_axes(&key_f32, &[0, 1, 3, 2]); + let scores = multiply_scalar(&matmul(&q_f32, &key_t), scale); + rows.push(matmul(&softmax(&scores, -1), &value_f32)); + } + + let mut rows = rows.into_iter(); + let mut result = rows.next().expect("reference batch must be non-empty"); + for row in rows { + result = concatenate(&result, &row, 0); + } + result +} + /// Deterministic, seed-varying FP32 values in roughly `[-1, 1]`. Avoids an RNG /// dependency while giving every decode step a fresh, non-degenerate q/k/v. fn pooled_pseudo_f32(seed: u64, n: usize) -> Vec { @@ -487,6 +438,26 @@ fn pooled_rms(a: &[f32], b: &[f32]) -> f32 { (sum_sq / a.len() as f64).sqrt() as f32 } +/// RMS error normalized by the dense reference's RMS. +/// +/// The pooled-vs-dense oracle needs a relative bound: an absolute `5e-3` +/// bound can accept a one-percent rescaling when the attention output itself +/// has RMS below 0.5. +fn pooled_normalized_rms(a: &[f32], b: &[f32]) -> f32 { + let diff_rms = pooled_rms(a, b); + let ref_rms = (b + .iter() + .map(|x| { + let x = f64::from(*x); + x * x + }) + .sum::() + / b.len() as f64) + .sqrt() + .max(1e-20) as f32; + diff_rms / ref_rms +} + /// A `PagedKvLayout` whose per-block byte budget is a positive placeholder; the /// real geometry is inferred from the first written block. fn pooled_layout( @@ -510,7 +481,8 @@ fn pooled_layout( /// NON-CONTIGUOUS physical rows by interleaving a spacer sequence's block /// writes between the target's. Each step appends one fresh token's K/V to both /// stores, picks a fresh pseudo-random `q [1, H, 1, D]`, runs the pooled and -/// dense fallbacks, and asserts the RMS of their outputs stays < 5e-3. +/// dense fallbacks, and asserts both absolute and normalized RMS stay below +/// `5e-3`. #[test] fn test_pooled_paged_decode_matches_dense_over_200_steps() { use crate::cache::{PagedBlockPool, PagedSequenceState}; @@ -534,6 +506,7 @@ fn test_pooled_paged_decode_matches_dense_over_200_steps() { let mut dense_v: Option> = None; let mut max_rms = 0.0f32; + let mut max_normalized_rms = 0.0f32; let mut prev_blocks = 0usize; for step in 0..STEPS { @@ -612,31 +585,23 @@ fn test_pooled_paged_decode_matches_dense_over_200_steps() { ) .unwrap(); - // --- dense fallback (identity block table) --- - let metadata = crate::cache::PagedDecodeMetadata::from_visible_lengths( - &[visible_len], - block_size as i32, - ) - .unwrap(); - let cache_keys = vec![dk.as_ref().unwrap() as *const MlxArray]; - let cache_values = vec![dv.as_ref().unwrap() as *const MlxArray]; - let dense_out = crate::layers::paged_decode_attention_dense_fallback( - &q, - &cache_keys, - &cache_values, - &metadata, - scale, - ) - .unwrap(); + // --- independent dense attention reference --- + let dense_out = dense_reference_decode_attention(&q, &[dk], &[dv], scale); let p = flatten_f32_local(&pooled_out); let d = flatten_f32_local(&dense_out); let rms = pooled_rms(&p, &d); + let normalized_rms = pooled_normalized_rms(&p, &d); assert!( rms < 5e-3, "step {step}: pooled vs dense RMS {rms} exceeded 5e-3" ); + assert!( + normalized_rms < 5e-3, + "step {step}: pooled vs dense normalized RMS {normalized_rms} exceeded 5e-3" + ); max_rms = max_rms.max(rms); + max_normalized_rms = max_normalized_rms.max(normalized_rms); } // The target eventually spans ceil(200/4) = 50 blocks; assert its block ids @@ -654,12 +619,18 @@ fn test_pooled_paged_decode_matches_dense_over_200_steps() { ); // Sanity: the run actually exercised a meaningful number of steps and the - // worst-case RMS is reported for the implementation summary. + // worst-case errors are reported for the implementation summary. assert!( max_rms < 5e-3, "max RMS {max_rms} over 200 steps exceeded 5e-3" ); - println!("test_pooled_paged_decode_matches_dense_over_200_steps: max RMS = {max_rms:e}"); + assert!( + max_normalized_rms < 5e-3, + "max normalized RMS {max_normalized_rms} over 200 steps exceeded 5e-3" + ); + println!( + "test_pooled_paged_decode_matches_dense_over_200_steps: max RMS = {max_rms:e}, max normalized RMS = {max_normalized_rms:e}" + ); } /// #123 fused-kernel parity: the native fused paged-attention decode kernel @@ -1130,8 +1101,8 @@ fn test_pooled_paged_decode_sliding_window_via_logical_start() { crate::layers::paged_decode_attention_pooled_fallback(&q, &pool, &states, layer_idx, scale) .unwrap(); - // Dense reference: slice the full dense buffer to the visible window and run - // the dense fallback over just those 7 tokens with identity metadata. + // Dense reference: slice the full dense buffer to the visible window and + // evaluate attention over just those 7 tokens. let dk = dense_k.as_ref().unwrap(); let dv = dense_v.as_ref().unwrap(); let window_k = slice( @@ -1144,29 +1115,20 @@ fn test_pooled_paged_decode_sliding_window_via_logical_start() { &[0, 0, window_start, 0], &[1, n_kv_heads, total, head_dim], ); - let visible_len = total - window_start; - let metadata = - crate::cache::PagedDecodeMetadata::from_visible_lengths(&[visible_len], block_size as i32) - .unwrap(); - let cache_keys = vec![window_k.as_ref().unwrap() as *const MlxArray]; - let cache_values = vec![window_v.as_ref().unwrap() as *const MlxArray]; - let dense_out = crate::layers::paged_decode_attention_dense_fallback( - &q, - &cache_keys, - &cache_values, - &metadata, - scale, - ) - .unwrap(); + let dense_out = dense_reference_decode_attention(&q, &[&window_k], &[&window_v], scale); - let rms = pooled_rms( - &flatten_f32_local(&pooled_out), - &flatten_f32_local(&dense_out), - ); + let pooled = flatten_f32_local(&pooled_out); + let dense = flatten_f32_local(&dense_out); + let rms = pooled_rms(&pooled, &dense); + let normalized_rms = pooled_normalized_rms(&pooled, &dense); assert!( rms < 5e-3, "sliding-window pooled vs dense RMS {rms} exceeded 5e-3" ); + assert!( + normalized_rms < 5e-3, + "sliding-window pooled vs dense normalized RMS {normalized_rms} exceeded 5e-3" + ); } /// Batch parity: a 2-sequence batch with different kv_lens, pooled vs dense, @@ -1237,34 +1199,23 @@ fn test_pooled_paged_decode_batch_of_two() { .unwrap(); assert_eq!(array_shape(&pooled_out), vec![2, n_kv_heads, 1, head_dim]); - // Dense fallback over the two dense buffers with identity metadata. - let metadata = - crate::cache::PagedDecodeMetadata::from_visible_lengths(&lens, block_size as i32).unwrap(); - let cache_keys = vec![ - dense_ks[0].as_ref().unwrap() as *const MlxArray, - dense_ks[1].as_ref().unwrap() as *const MlxArray, - ]; - let cache_values = vec![ - dense_vs[0].as_ref().unwrap() as *const MlxArray, - dense_vs[1].as_ref().unwrap() as *const MlxArray, - ]; - let dense_out = crate::layers::paged_decode_attention_dense_fallback( - &q, - &cache_keys, - &cache_values, - &metadata, - scale, - ) - .unwrap(); + // Independent dense attention over both rows, whose visible lengths differ. + let dense_keys: Vec<&MlxArray> = dense_ks.iter().map(|key| &**key).collect(); + let dense_values: Vec<&MlxArray> = dense_vs.iter().map(|value| &**value).collect(); + let dense_out = dense_reference_decode_attention(&q, &dense_keys, &dense_values, scale); - let rms = pooled_rms( - &flatten_f32_local(&pooled_out), - &flatten_f32_local(&dense_out), - ); + let pooled = flatten_f32_local(&pooled_out); + let dense = flatten_f32_local(&dense_out); + let rms = pooled_rms(&pooled, &dense); + let normalized_rms = pooled_normalized_rms(&pooled, &dense); assert!( rms < 5e-3, "batch-of-two pooled vs dense RMS {rms} exceeded 5e-3" ); + assert!( + normalized_rms < 5e-3, + "batch-of-two pooled vs dense normalized RMS {normalized_rms} exceeded 5e-3" + ); } /// End-to-end prefill -> decode parity (#120 acceptance criterion 1). @@ -1357,6 +1308,7 @@ fn test_pooled_prefill_then_decode_matches_dense() { assert_eq!(target.layer(layer_idx).unwrap().block_ids.len(), 4); let mut max_rms = 0.0f32; + let mut max_normalized_rms = 0.0f32; for step in 0..STEPS { let t = PROMPT + step as i32; // absolute token index being appended @@ -1403,34 +1355,27 @@ fn test_pooled_prefill_then_decode_matches_dense() { ) .unwrap(); - let metadata = crate::cache::PagedDecodeMetadata::from_visible_lengths( - &[visible_len], - block_size as i32, - ) - .unwrap(); - let cache_keys = vec![dk.as_ref().unwrap() as *const MlxArray]; - let cache_values = vec![dv.as_ref().unwrap() as *const MlxArray]; - let dense_out = crate::layers::paged_decode_attention_dense_fallback( - &q, - &cache_keys, - &cache_values, - &metadata, - scale, - ) - .unwrap(); + let dense_out = dense_reference_decode_attention(&q, &[dk], &[dv], scale); - let rms = pooled_rms( - &flatten_f32_local(&pooled_out), - &flatten_f32_local(&dense_out), - ); + let pooled = flatten_f32_local(&pooled_out); + let dense = flatten_f32_local(&dense_out); + let rms = pooled_rms(&pooled, &dense); + let normalized_rms = pooled_normalized_rms(&pooled, &dense); assert!( rms < 5e-3, "prefill->decode step {step} (abs token {t}): pooled vs dense RMS {rms} exceeded 5e-3" ); + assert!( + normalized_rms < 5e-3, + "prefill->decode step {step} (abs token {t}): pooled vs dense normalized RMS {normalized_rms} exceeded 5e-3" + ); max_rms = max_rms.max(rms); + max_normalized_rms = max_normalized_rms.max(normalized_rms); } - println!("test_pooled_prefill_then_decode_matches_dense: max RMS = {max_rms:e}"); + println!( + "test_pooled_prefill_then_decode_matches_dense: max RMS = {max_rms:e}, max normalized RMS = {max_normalized_rms:e}" + ); } /// Flatten any tensor to a row-major `Vec` (after an FP32 cast). Local to @@ -4258,36 +4203,15 @@ fn test_batch_quantized_kv_cache_make_mask_no_padding_multi_token() { ); } -/// Issue #195: the paged-decode attention fallbacks must reject an empty -/// batch with a clean `Err` instead of panicking in `drain(..1)`. +/// Issue #195: the pooled paged-decode fallback must reject an empty batch +/// with a clean `Err` instead of panicking while assembling row outputs. #[test] -fn paged_decode_fallbacks_reject_an_empty_batch() { - use crate::cache::{ - PagedBlockPool, PagedDecodeMetadata, PagedKvLayout, PagedSequenceState, - RotatingPagedDecodeMetadata, - }; +fn paged_decode_pooled_fallback_rejects_an_empty_batch() { + use crate::cache::{PagedBlockPool, PagedKvLayout, PagedSequenceState}; // A valid 4-D decode query shape with batch == 0. let q = zeros(&[0, 1, 1, 2], crate::dtype::FLOAT32); - let dense_meta = PagedDecodeMetadata::from_visible_lengths(&[], 2).unwrap(); - let dense = - crate::layers::paged_decode_attention_dense_fallback(&q, &[], &[], &dense_meta, 1.0); - assert!( - dense.as_ref().is_err_and(|e| e.contains("empty batch")), - "dense fallback must reject batch == 0, got error {:?}", - dense.as_ref().err() - ); - - let rot_meta = RotatingPagedDecodeMetadata::from_parts(&[], &[], 2).unwrap(); - let rotating = - crate::layers::paged_decode_attention_rotating_fallback(&q, &[], &[], &rot_meta, 1.0); - assert!( - rotating.as_ref().is_err_and(|e| e.contains("empty batch")), - "rotating fallback must reject batch == 0, got error {:?}", - rotating.as_ref().err() - ); - let layout = PagedKvLayout::uniform(1, 2, 32).unwrap(); let pool = PagedBlockPool::new(layout); let states: Vec<&PagedSequenceState> = Vec::new(); diff --git a/src/lib/mlxcel-core/src/layers.rs b/src/lib/mlxcel-core/src/layers.rs index d136c3543..61ed1c377 100644 --- a/src/lib/mlxcel-core/src/layers.rs +++ b/src/lib/mlxcel-core/src/layers.rs @@ -5349,173 +5349,6 @@ pub unsafe fn attention_from_ptr( attention(q, k, v, scale, mask, softcap, window_size) } -fn validate_paged_decode_inputs( - q: &MlxArray, - cache_keys: &[*const MlxArray], - cache_values: &[*const MlxArray], - metadata: &crate::cache::PagedDecodeMetadata, -) -> Result<(), String> { - let q_shape = ffi::array_shape(q); - if q_shape.len() != 4 { - return Err(format!( - "paged decode attention expected q rank 4, got shape {:?}", - q_shape - )); - } - if q_shape[2] != 1 { - return Err(format!( - "paged decode attention only supports decode-only q_len == 1, got {}", - q_shape[2] - )); - } - - let batch = q_shape[0].max(0) as usize; - if cache_keys.len() != batch || cache_values.len() != batch { - return Err(format!( - "paged decode attention expected {} cache pointers, got {} keys and {} values", - batch, - cache_keys.len(), - cache_values.len() - )); - } - if cache_keys.iter().any(|ptr| ptr.is_null()) || cache_values.iter().any(|ptr| ptr.is_null()) { - return Err( - "paged decode attention received a null dense compatibility cache pointer".to_string(), - ); - } - if metadata.len() != batch { - return Err(format!( - "paged decode attention expected {} metadata entries, got {}", - batch, - metadata.len() - )); - } - if metadata.block_table_offsets.len() != batch + 1 { - return Err(format!( - "paged decode attention expected {} block offsets, got {}", - batch + 1, - metadata.block_table_offsets.len() - )); - } - if metadata - .block_table_offsets - .last() - .copied() - .unwrap_or_default() as usize - != metadata.block_tables.len() - { - return Err( - "paged decode attention block table offsets do not cover the flattened block table" - .to_string(), - ); - } - - Ok(()) -} - -/// Reference paged decode path over dense compatibility KV caches. -/// -/// This stays entirely in Rust/FFI wrappers and is primarily used as a -/// correctness baseline and benchmark fallback for the native paged decode -/// kernel. -pub fn paged_decode_attention_dense_fallback( - q: &MlxArray, - cache_keys: &[*const MlxArray], - cache_values: &[*const MlxArray], - metadata: &crate::cache::PagedDecodeMetadata, - scale: f32, -) -> Result, String> { - validate_paged_decode_inputs(q, cache_keys, cache_values, metadata)?; - - let q_shape = ffi::array_shape(q); - let batch = q_shape[0].max(0) as usize; - let mut outputs: Vec> = Vec::with_capacity(batch); - - for batch_idx in 0..batch { - let kv_len = metadata.kv_lens[batch_idx]; - if kv_len <= 0 { - return Err(format!( - "paged decode fallback requires kv_len > 0 for batch index {batch_idx}, got {kv_len}" - )); - } - - let q_i = ffi::slice( - q, - &[batch_idx as i32, 0, 0, 0], - &[batch_idx as i32 + 1, i32::MAX, 1, i32::MAX], - ); - - let table_begin = metadata.block_table_offsets[batch_idx] as usize; - let table_end = metadata.block_table_offsets[batch_idx + 1] as usize; - let key_cache = unsafe { &*cache_keys[batch_idx] }; - let value_cache = unsafe { &*cache_values[batch_idx] }; - - let mut key_visible: Option> = None; - let mut value_visible: Option> = None; - - for &logical_block in &metadata.block_tables[table_begin..table_end] { - let block_start = logical_block * metadata.block_size; - if block_start >= kv_len { - continue; - } - let block_end = (block_start + metadata.block_size).min(kv_len); - - let key_block = ffi::slice( - key_cache, - &[0, 0, block_start, 0], - &[1, i32::MAX, block_end, i32::MAX], - ); - let value_block = ffi::slice( - value_cache, - &[0, 0, block_start, 0], - &[1, i32::MAX, block_end, i32::MAX], - ); - - key_visible = Some(match key_visible { - Some(prev) => crate::concatenate(&prev, &key_block, 2), - None => key_block, - }); - value_visible = Some(match value_visible { - Some(prev) => crate::concatenate(&prev, &value_block, 2), - None => value_block, - }); - } - - let key_visible = key_visible.ok_or_else(|| { - format!("paged decode fallback built no visible key blocks for batch index {batch_idx}") - })?; - let value_visible = value_visible.ok_or_else(|| { - format!( - "paged decode fallback built no visible value blocks for batch index {batch_idx}" - ) - })?; - - outputs.push(unsafe { - attention_from_ptr( - &q_i, - &key_visible, - &value_visible, - scale, - std::ptr::null(), - 0.0, - 0, - ) - }); - } - - // `drain(..1)` panics on an empty vec, so reject `batch == 0` explicitly - // with a clean error instead (issue #195). - if outputs.is_empty() { - return Err("paged decode fallback received an empty batch (batch == 0)".to_string()); - } - let mut outputs = outputs.into_iter(); - let mut result = outputs.next().expect("outputs is non-empty: checked above"); - for output in outputs { - result = crate::concatenate(&result, &output, 0); - } - Ok(result) -} - fn validate_pooled_paged_decode_inputs( q: &MlxArray, states: &[&crate::cache::PagedSequenceState], @@ -5548,14 +5381,10 @@ fn validate_pooled_paged_decode_inputs( /// Reference paged decode path that gathers K/V from the [`PagedBlockPool`]. /// /// This is the Phase 2 (#119) pooled read path of the unified paged KV cache -/// (epic #116). It is the pooled analogue of [`paged_decode_attention_dense_fallback`], -/// which stays the live decode path and the parity baseline this function is -/// validated against. The two differ only in where the per-sequence visible -/// K/V comes from: the dense fallback slices contiguous dense compatibility -/// buffers, while this function calls [`crate::cache::PagedBlockPool::gather_visible`] -/// to pull each sequence's physical blocks out of the pool by block-table order. -/// Everything downstream — the per-batch `q` slice, the fused SDPA dispatch via -/// [`attention_from_ptr`], and the axis-0 batch concat — is identical. +/// (epic #116). It calls [`crate::cache::PagedBlockPool::gather_visible`] to +/// pull each sequence's physical blocks out of the pool by block-table order, +/// then runs the per-batch `q` slice through fused SDPA and concatenates the +/// row outputs on axis 0. /// /// Per ADR 0001 (`docs/adr/0001-paged-attention-gather-vs-fused-kernel.md`) this /// is strategy option A, *gather-then-SDPA*: `gather_visible` builds a @@ -5574,8 +5403,7 @@ fn validate_pooled_paged_decode_inputs( /// `trim_tokens` advances as the window slides), so `gather_visible` returns /// exactly the `[logical_start, len)` window already shaped /// `[1, n_kv_heads, visible_len, head_dim]` (SDPA-ready). That is why there is -/// no separate "rotating" pooled variant mirroring -/// [`paged_decode_attention_rotating_fallback`]. +/// no separate rotating pooled variant for ring-buffer cache storage. /// /// `gather_visible` returning `Ok(None)` (no visible tokens, or the layer has no /// pool storage yet) is treated as an error here, because decode requires a @@ -5763,10 +5591,9 @@ pub fn select_pooled_paged_dispatch( /// values (`0` / `false` / `off` / `no`) as its kill switch: setting it /// restores the pre-#899 gather-then-SDPA behaviour end to end. /// -/// The dense-compat block-table entries (`paged_decode_attention_dense_compat` -/// and `_rotating_compat`) are not governed by it; since #2172 no production -/// forward calls them (the model-owned families attend through their caches, -/// ADR 0009), and they remain as the reference pair their FFI tests pin. +/// Model-owned families are not governed by it: since #2172 they attend +/// directly through their dense cache implementations (ADR 0009), without a +/// pooled or block-table decode route. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum NativePagedOverride { /// Force the fused kernel, bypassing the adaptive selector (the original @@ -6019,10 +5846,8 @@ fn resolve_pooled_paged_dispatch( /// entry point to a library API (ADR 0001, resolving issue #710). The in-server /// decode routes pool-backed layers through the per-sequence `update_and_fetch` /// pool intercept (gather + standard SDPA, `src/models/llama3.rs` -/// `is_paged_backed()`), and production models call -/// [`paged_decode_attention_dense_compat`] / [`paged_decode_attention_dense_fallback`], -/// never this function. It is kept and tested for external mlxcel-core consumers -/// and `examples/paged_attention_kernel_bench.rs`. ADR 0001 records the +/// `is_paged_backed()`). It is kept and tested for external mlxcel-core +/// consumers and `examples/paged_attention_kernel_bench.rs`. ADR 0001 records the /// occupancy derivation: the selector's single-slab batched island (`B>=4`, at /// most ~256 tokens per sequence at `block_size` 32) is unreachable under the /// shipped serving defaults and transient even when entered, so a scheduler @@ -6063,246 +5888,6 @@ pub fn paged_decode_attention_pooled( paged_decode_attention_pooled_fallback(q, pool, states, layer_idx, scale) } -/// Native paged decode path over dense compatibility KV caches. -/// -/// The C++ bridge consumes the logical block table metadata and performs the -/// per-sequence block gathering plus SDPA dispatch natively, reducing Rust-side -/// FFI churn on the decode hot path. -pub fn paged_decode_attention_dense_compat( - q: &MlxArray, - cache_keys: &[*const MlxArray], - cache_values: &[*const MlxArray], - metadata: &crate::cache::PagedDecodeMetadata, - scale: f32, -) -> Result, String> { - validate_paged_decode_inputs(q, cache_keys, cache_values, metadata)?; - Ok(unsafe { - ffi::paged_decode_attention_dense_compat( - q, - cache_keys, - cache_values, - &metadata.kv_lens, - &metadata.block_tables, - &metadata.block_table_offsets, - metadata.block_size, - scale, - ) - }) -} - -fn validate_rotating_paged_decode_inputs( - q: &MlxArray, - cache_keys: &[*const MlxArray], - cache_values: &[*const MlxArray], - metadata: &crate::cache::RotatingPagedDecodeMetadata, -) -> Result<(), String> { - let q_shape = ffi::array_shape(q); - if q_shape.len() != 4 { - return Err(format!( - "rotating paged decode attention expected q rank 4, got shape {:?}", - q_shape - )); - } - if q_shape[2] != 1 { - return Err(format!( - "rotating paged decode attention only supports decode-only q_len == 1, got {}", - q_shape[2] - )); - } - - let batch = q_shape[0].max(0) as usize; - if cache_keys.len() != batch || cache_values.len() != batch { - return Err(format!( - "rotating paged decode attention expected {} cache pointers, got {} keys and {} values", - batch, - cache_keys.len(), - cache_values.len() - )); - } - if cache_keys.iter().any(|ptr| ptr.is_null()) || cache_values.iter().any(|ptr| ptr.is_null()) { - return Err( - "rotating paged decode attention received a null ring-buffer cache pointer".to_string(), - ); - } - if metadata.len() != batch { - return Err(format!( - "rotating paged decode attention expected {} metadata entries, got {}", - batch, - metadata.len() - )); - } - Ok(()) -} - -/// Reference paged decode path over rotating ring-buffer KV caches. -pub fn paged_decode_attention_rotating_fallback( - q: &MlxArray, - cache_keys: &[*const MlxArray], - cache_values: &[*const MlxArray], - metadata: &crate::cache::RotatingPagedDecodeMetadata, - scale: f32, -) -> Result, String> { - validate_rotating_paged_decode_inputs(q, cache_keys, cache_values, metadata)?; - - let q_shape = ffi::array_shape(q); - let batch = q_shape[0].max(0) as usize; - let mut outputs: Vec> = Vec::with_capacity(batch); - - for batch_idx in 0..batch { - let kv_len = metadata.kv_lens[batch_idx]; - let logical_start = metadata.logical_starts[batch_idx]; - if kv_len <= 0 { - return Err(format!( - "rotating paged decode fallback requires kv_len > 0 for batch index {batch_idx}, got {kv_len}" - )); - } - - let q_i = ffi::slice( - q, - &[batch_idx as i32, 0, 0, 0], - &[batch_idx as i32 + 1, i32::MAX, 1, i32::MAX], - ); - - let key_cache = unsafe { &*cache_keys[batch_idx] }; - let value_cache = unsafe { &*cache_values[batch_idx] }; - let buffer_len = ffi::array_shape(key_cache) - .get(2) - .copied() - .unwrap_or_default(); - if buffer_len <= 0 { - return Err(format!( - "rotating paged decode fallback requires non-empty buffer for batch index {batch_idx}" - )); - } - if logical_start < 0 || logical_start >= buffer_len { - return Err(format!( - "rotating paged decode fallback received invalid logical_start={logical_start} for buffer_len={buffer_len}" - )); - } - - let block_count = (kv_len + metadata.block_size - 1) / metadata.block_size; - let mut key_visible: Option> = None; - let mut value_visible: Option> = None; - for logical_block in 0..block_count { - let logical_pos = logical_block * metadata.block_size; - let logical_end = (logical_pos + metadata.block_size).min(kv_len); - let token_count = logical_end - logical_pos; - if token_count <= 0 { - continue; - } - - let physical_start = (logical_start + logical_pos).rem_euclid(buffer_len); - let physical_end = physical_start + token_count; - let key_block = if physical_end <= buffer_len { - ffi::slice( - key_cache, - &[0, 0, physical_start, 0], - &[1, i32::MAX, physical_end, i32::MAX], - ) - } else { - let key_tail = ffi::slice( - key_cache, - &[0, 0, physical_start, 0], - &[1, i32::MAX, buffer_len, i32::MAX], - ); - let key_head = ffi::slice( - key_cache, - &[0, 0, 0, 0], - &[1, i32::MAX, physical_end - buffer_len, i32::MAX], - ); - crate::concatenate(&key_tail, &key_head, 2) - }; - let value_block = if physical_end <= buffer_len { - ffi::slice( - value_cache, - &[0, 0, physical_start, 0], - &[1, i32::MAX, physical_end, i32::MAX], - ) - } else { - let value_tail = ffi::slice( - value_cache, - &[0, 0, physical_start, 0], - &[1, i32::MAX, buffer_len, i32::MAX], - ); - let value_head = ffi::slice( - value_cache, - &[0, 0, 0, 0], - &[1, i32::MAX, physical_end - buffer_len, i32::MAX], - ); - crate::concatenate(&value_tail, &value_head, 2) - }; - - key_visible = Some(match key_visible { - Some(prev) => crate::concatenate(&prev, &key_block, 2), - None => key_block, - }); - value_visible = Some(match value_visible { - Some(prev) => crate::concatenate(&prev, &value_block, 2), - None => value_block, - }); - } - - let key_visible = key_visible.ok_or_else(|| { - format!( - "rotating paged decode fallback built no visible key blocks for batch index {batch_idx}" - ) - })?; - let value_visible = value_visible.ok_or_else(|| { - format!( - "rotating paged decode fallback built no visible value blocks for batch index {batch_idx}" - ) - })?; - - outputs.push(unsafe { - attention_from_ptr( - &q_i, - &key_visible, - &value_visible, - scale, - std::ptr::null(), - 0.0, - 0, - ) - }); - } - - // `drain(..1)` panics on an empty vec, so reject `batch == 0` explicitly - // with a clean error instead (issue #195). - if outputs.is_empty() { - return Err( - "rotating paged decode fallback received an empty batch (batch == 0)".to_string(), - ); - } - let mut outputs = outputs.into_iter(); - let mut result = outputs.next().expect("outputs is non-empty: checked above"); - for output in outputs { - result = crate::concatenate(&result, &output, 0); - } - Ok(result) -} - -/// Native paged decode path over rotating ring-buffer KV caches. -pub fn paged_decode_attention_rotating_compat( - q: &MlxArray, - cache_keys: &[*const MlxArray], - cache_values: &[*const MlxArray], - metadata: &crate::cache::RotatingPagedDecodeMetadata, - scale: f32, -) -> Result, String> { - validate_rotating_paged_decode_inputs(q, cache_keys, cache_values, metadata)?; - Ok(unsafe { - ffi::paged_decode_attention_rotating_compat( - q, - cache_keys, - cache_values, - &metadata.kv_lens, - &metadata.logical_starts, - metadata.block_size, - scale, - ) - }) -} - /// Metal 4 fused attention dispatch with automatic hardware detection. /// /// Dispatches SDPA to the Metal 4 fused kernel when the hardware supports it, diff --git a/src/lib/mlxcel-core/src/lib.rs b/src/lib/mlxcel-core/src/lib.rs index 5a09c6447..cd7735d5d 100644 --- a/src/lib/mlxcel-core/src/lib.rs +++ b/src/lib/mlxcel-core/src/lib.rs @@ -1388,36 +1388,6 @@ mod ffi { scale: f32, ) -> UniquePtr; - /// Decode-only paged attention over dense compatibility KV caches. - /// - /// `q` is `[B, Hq, 1, D]`. `cache_keys[i]` / `cache_values[i]` are - /// per-sequence dense KV buffers, while `kv_lens`, `block_tables`, and - /// `block_table_offsets` describe the logical paged layout. - unsafe fn paged_decode_attention_dense_compat( - q: &MlxArray, - cache_keys: &[*const MlxArray], - cache_values: &[*const MlxArray], - kv_lens: &[i32], - block_tables: &[i32], - block_table_offsets: &[i32], - block_size: i32, - scale: f32, - ) -> UniquePtr; - - /// Decode-only paged attention over rotating ring-buffer KV caches. - /// - /// `logical_starts[i]` identifies the physical start of the logical - /// visible window within `cache_keys[i]` / `cache_values[i]`. - unsafe fn paged_decode_attention_rotating_compat( - q: &MlxArray, - cache_keys: &[*const MlxArray], - cache_values: &[*const MlxArray], - kv_lens: &[i32], - logical_starts: &[i32], - block_size: i32, - scale: f32, - ) -> UniquePtr; - /// Fused paged-attention decode Metal kernel (epic #116 Phase 6, #123). /// /// Reads scattered KV blocks directly out of the global pool via the From fb2cb7832028c7425dc347c23cc4fdaeb1d64d7c Mon Sep 17 00:00:00 2001 From: Jeongkyu Shin Date: Sun, 11 Oct 2026 12:13:28 +0900 Subject: [PATCH 2/7] fix: harden storage benchmark provenance Record both the configured MLX pin and the release build's actual MLX checkout, rejecting a mismatch before measurement. Check that the serving port is free and reuse the repository hostgate before every fresh-server arm, preserving per-arm wait, CI, process, memory, load, and driver evidence for the multi-hour run. Refs #2230 --- .../gate.py | 76 +++++++++++++++++++ .../run.sh | 23 +++++- 2 files changed, 97 insertions(+), 2 deletions(-) create mode 100755 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/gate.py diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/gate.py b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/gate.py new file mode 100755 index 000000000..21ab65e50 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/gate.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Run the repository's quiet-host gate and record per-arm evidence.""" + +from __future__ import annotations + +import argparse +import datetime as dt +import importlib +import json +import os +import socket +import sys +from pathlib import Path + + +def assert_port_free(port: int) -> None: + """Fail if the benchmark's IPv4 loopback port is already in use.""" + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + try: + sock.bind(("127.0.0.1", port)) + except OSError as exc: + raise SystemExit(f"benchmark port {port} is already in use: {exc}") from exc + + +def utc_now() -> str: + """Return the current UTC time in the environment record's format.""" + + return dt.datetime.now(dt.UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def main() -> int: + """Gate one arm, fail closed on port conflicts, and append its evidence.""" + + parser = argparse.ArgumentParser() + parser.add_argument("--hostgate-dir", type=Path, required=True) + parser.add_argument("--evidence", type=Path, required=True) + parser.add_argument("--label", required=True) + parser.add_argument("--port", type=int, required=True) + parser.add_argument("--port-only", action="store_true") + args = parser.parse_args() + + started = utc_now() + assert_port_free(args.port) + record: dict[str, object] = { + "label": args.label, + "started_utc": started, + "port": args.port, + "port_free_before_wait": True, + } + + if not args.port_only: + sys.path.insert(0, str(args.hostgate_dir)) + gate = importlib.import_module("hostgate") + record["gate_wait_s"] = gate.wait_quiet(log=sys.stderr) + assert_port_free(args.port) + record.update( + port_free_after_wait=True, + ci_job_running=gate.ci_job_running(), + busy_procs=gate.busy_procs(), + foreign_model_procs=gate.foreign_model_procs(), + load1=os.getloadavg()[0], + mem_available_gib=gate.mem_available_gib(), + ) + nvrm_window, nvrm_total = gate.nvrm_counts() + record.update(nvrm_window_5m=nvrm_window, nvrm_total_since_boot=nvrm_total) + + record["completed_utc"] = utc_now() + args.evidence.parent.mkdir(parents=True, exist_ok=True) + with args.evidence.open("a", encoding="utf-8") as output: + output.write(json.dumps(record, sort_keys=True) + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/run.sh b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/run.sh index 3654f5b67..08ee101db 100755 --- a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/run.sh +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/run.sh @@ -6,10 +6,13 @@ BIN=/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/targe CLIENT=/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/scripts/bench_serving_concurrency.py MODEL_ROOT=/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/models/mlx DATA=/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11 +GATE_DRIVER=${DATA}/gate.py +HOSTGATE_DIR=${WORKTREE}/docs/benchmark_results/data/sdpa-plan-bucket-gb10-2026-09-12/harness PORT=18080 export MLXCEL_CCCL_DIR=/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/target/release/build/mlxcel-core-4c8821fbbcc3d8f4/out/build/include/cccl export MLXCEL_CUTLASS_DIR=/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/target/release/build/mlxcel-core-4c8821fbbcc3d8f4/out/build/include +MLX_SOURCE_DIR=/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/target/release/build/mlxcel-core-4c8821fbbcc3d8f4/out/build/_deps/mlx-src SERVER_PID= cleanup_server() { @@ -24,7 +27,7 @@ trap cleanup_server EXIT INT TERM mkdir -p "${DATA}" exec > >(tee -a "${DATA}/run.log") 2>&1 -for required in "${BIN}" "${CLIENT}" "${MODEL_ROOT}/gemma-3-1b-it-4bit" "${MODEL_ROOT}/llama-4-scout-17b-16e-instruct-4bit" "${MLXCEL_CCCL_DIR}" "${MLXCEL_CUTLASS_DIR}"; do +for required in "${BIN}" "${CLIENT}" "${GATE_DRIVER}" "${HOSTGATE_DIR}/hostgate.py" "${MODEL_ROOT}/gemma-3-1b-it-4bit" "${MODEL_ROOT}/llama-4-scout-17b-16e-instruct-4bit" "${MLXCEL_CCCL_DIR}" "${MLXCEL_CUTLASS_DIR}" "${MLX_SOURCE_DIR}/.git"; do if [[ ! -e "${required}" ]]; then echo "missing required benchmark path: ${required}" >&2 exit 2 @@ -36,13 +39,27 @@ if [[ -e "${DATA}/complete" ]]; then exit 2 fi +: > "${DATA}/hostgate.jsonl" +python3 "${GATE_DRIVER}" --hostgate-dir "${HOSTGATE_DIR}" --evidence "${DATA}/hostgate.jsonl" --label run-preflight --port "${PORT}" --port-only + +mlx_cmake_tag=$(awk '$1 == "GIT_TAG" { gsub(/[")]/, "", $2); print $2; exit }' "${WORKTREE}/src/lib/mlx-cpp/CMakeLists.txt") +mlx_checkout_commit=$(git -C "${MLX_SOURCE_DIR}" rev-parse HEAD) +if [[ -z "${mlx_cmake_tag}" || "${mlx_cmake_tag}" != "${mlx_checkout_commit}" ]]; then + echo "MLX pin mismatch: CMake GIT_TAG=${mlx_cmake_tag:-missing}, release checkout=${mlx_checkout_commit}" >&2 + exit 2 +fi + { echo "started_utc=$(date -u +%FT%TZ)" echo "worktree=${WORKTREE}" echo "binary=${BIN}" echo "binary_sha256=$(sha256sum "${BIN}" | awk '{print $1}')" echo "commit=$(git -C "${WORKTREE}" rev-parse HEAD)" - echo "mlx_pin=$(git -C "${WORKTREE}/src/lib/mlx-cpp" rev-parse HEAD)" + echo "mlx_cmake_git_tag=${mlx_cmake_tag}" + echo "mlx_checkout=${MLX_SOURCE_DIR}" + echo "mlx_checkout_commit=${mlx_checkout_commit}" + echo "hostgate_source=${HOSTGATE_DIR}/hostgate.py" + echo "hostgate_sha256=$(sha256sum "${HOSTGATE_DIR}/hostgate.py" | awk '{print $1}')" echo "host=$(hostname)" uname -a nvidia-smi --query-gpu=name,driver_version,pci.bus_id --format=csv,noheader @@ -81,6 +98,8 @@ run_arm() { local server_log="${DATA}/server-${stem}.log" local client_log="${DATA}/client-${stem}.txt" + echo "GATE ${stem} $(date -u +%FT%TZ)" + python3 "${GATE_DRIVER}" --hostgate-dir "${HOSTGATE_DIR}" --evidence "${DATA}/hostgate.jsonl" --label "${stem}" --port "${PORT}" printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "${model_name}" "${prompt_tokens}" "${round}" "${position}" "${arm}" "${backend}" "$(date -u +%FT%TZ)" >> "${DATA}/schedule.tsv" echo "START ${stem} $(date -u +%FT%TZ)" "${BIN}" serve -m "${model_path}" --port "${PORT}" --metrics --parallel 8 --decode-storage-backend "${backend}" > "${server_log}" 2>&1 & From 35da2dbe32c731ff6645b2b32a2117af2965f332 Mon Sep 17 00:00:00 2001 From: Jeongkyu Shin Date: Sun, 11 Oct 2026 13:19:11 +0900 Subject: [PATCH 3/7] fix: validate storage benchmark protocol Retry the host gate when activity appears after the quiet wait, and refuse to record any nonquiet arm. Require complete provenance, exact ordered schedules and logs, quiet evidence, and backend markers before computing the paired/null storage decision. Add lightweight fixtures for valid evidence plus missing, dirty, reordered, nonmonotonic, and substituted protocol artifacts. Refs #2230 --- .../analyze.py | 553 ++++++++++++++++-- .../gate.py | 97 ++- .../run.sh | 35 +- .../test_protocol.py | 309 ++++++++++ 4 files changed, 927 insertions(+), 67 deletions(-) create mode 100755 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/test_protocol.py diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/analyze.py b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/analyze.py index 813f5f352..81d9ac856 100755 --- a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/analyze.py +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/analyze.py @@ -1,15 +1,30 @@ #!/usr/bin/env python3 -"""Summarize the model-owned decode-storage paired/null serving sweep.""" +"""Validate and summarize the model-owned storage paired/null serving sweep.""" from __future__ import annotations +import argparse +import csv +import datetime as dt +import hashlib +import json +import math import re import statistics +import subprocess from dataclasses import dataclass from pathlib import Path +from typing import Any DATA = Path(__file__).resolve().parent +MODELS = ("gemma3", "llama4") +PROMPTS = (512, 4096) +ROUNDS = range(1, 6) +ARMS = ("paged-a", "dense", "paged-b") +CONCURRENCIES = (1, 4, 8) +HEX40 = re.compile(r"[0-9a-f]{40}") +HEX64 = re.compile(r"[0-9a-f]{64}") LOG_NAME = re.compile( r"client-(?Pgemma3|llama4)-p(?P512|4096)-" r"r(?P[1-5])-(?Ppaged-a|dense|paged-b)\.txt" @@ -18,6 +33,37 @@ r"^\s*(?P1|4|8)\s+\d+\s+(?P\d+)\s+" r"(?:\d+(?:\.\d+)?\s+){3}(?P\d+(?:\.\d+)?)\s*$" ) +SCHEDULE_FIELDS = ( + "model", + "prompt_tokens", + "round", + "position", + "arm", + "backend", + "started_utc", +) + + +@dataclass(frozen=True) +class ArmSpec: + """One of the protocol's 60 fresh-server arms.""" + + model: str + prompt_tokens: int + round: int + position: int + arm: str + backend: str + + @property + def label(self) -> str: + return ( + f"{self.model}-p{self.prompt_tokens}-r{self.round}-{self.arm}" + ) + + @property + def client_name(self) -> str: + return f"client-{self.label}.txt" @dataclass(frozen=True) @@ -32,81 +78,487 @@ class Run: aggregate_tok_s: float -def percent_delta(value: float, baseline: float) -> float: - """Return the percentage delta of ``value`` from ``baseline``.""" +@dataclass(frozen=True) +class Protocol: + """Validated evidence needed before a storage decision is permitted.""" + + environment: dict[str, str] + schedule_times: dict[str, dt.datetime] + runs: tuple[Run, ...] + + +def arm_order(round_no: int) -> tuple[str, str, str]: + """Return the preregistered rotating arm order for a round.""" + + orders = ( + ("paged-a", "dense", "paged-b"), + ("dense", "paged-b", "paged-a"), + ("paged-b", "paged-a", "dense"), + ) + return orders[(round_no - 1) % len(orders)] + + +def expected_arms() -> tuple[ArmSpec, ...]: + """Return the exact ordered 60-arm protocol.""" + + specs = [] + for model in MODELS: + for prompt in PROMPTS: + for round_no in ROUNDS: + for position, arm in enumerate(arm_order(round_no), start=1): + specs.append( + ArmSpec( + model=model, + prompt_tokens=prompt, + round=round_no, + position=position, + arm=arm, + backend="dense" if arm == "dense" else "paged", + ) + ) + return tuple(specs) + + +def parse_utc(value: object, context: str) -> dt.datetime: + """Parse one second-resolution UTC timestamp and reject ambiguous forms.""" + + if not isinstance(value, str) or not value.endswith("Z"): + raise ValueError(f"{context}: expected a UTC timestamp ending in Z") + try: + parsed = dt.datetime.fromisoformat(value[:-1] + "+00:00") + except ValueError as exc: + raise ValueError(f"{context}: invalid UTC timestamp {value!r}") from exc + if parsed.microsecond != 0: + raise ValueError(f"{context}: timestamp must use whole seconds") + return parsed + + +def sha256_file(path: Path) -> str: + """Return a streaming SHA-256 for a provenance artifact.""" + + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def read_environment(data: Path) -> tuple[dict[str, str], dt.datetime, dt.datetime]: + """Validate source, binary, MLX, host, and script provenance.""" + + path = data / "environment.txt" + if not path.is_file(): + raise ValueError(f"missing provenance file: {path}") + environment: dict[str, str] = {} + for line_no, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + if not line: + continue + key, separator, value = line.partition("=") + if not separator or not key or key in environment: + raise ValueError(f"{path}:{line_no}: malformed or duplicate provenance key") + environment[key] = value + + required = { + "started_utc", + "completed_utc", + "worktree", + "worktree_clean", + "binary", + "binary_sha256", + "binary_size_bytes", + "binary_mtime_utc", + "commit", + "mlx_cmake_git_tag", + "mlx_checkout", + "mlx_checkout_commit", + "hostgate_source", + "hostgate_sha256", + "gate_sha256", + "analyzer_sha256", + "run_sha256", + "host", + "uname", + "gpu", + } + missing = sorted(required - environment.keys()) + if missing: + raise ValueError(f"{path}: missing provenance keys: {', '.join(missing)}") + + started = parse_utc(environment["started_utc"], f"{path}:started_utc") + completed = parse_utc(environment["completed_utc"], f"{path}:completed_utc") + if completed <= started: + raise ValueError(f"{path}: completed_utc must be after started_utc") + if environment["worktree_clean"] != "true": + raise ValueError(f"{path}: benchmark did not start from a clean worktree") + if HEX40.fullmatch(environment["commit"]) is None: + raise ValueError(f"{path}: commit is not a 40-hex object id") + if HEX40.fullmatch(environment["mlx_cmake_git_tag"]) is None: + raise ValueError(f"{path}: MLX CMake GIT_TAG is not a 40-hex object id") + if HEX40.fullmatch(environment["mlx_checkout_commit"]) is None: + raise ValueError(f"{path}: MLX checkout commit is not a 40-hex object id") + if environment["mlx_cmake_git_tag"] != environment["mlx_checkout_commit"]: + raise ValueError(f"{path}: configured and built MLX commits differ") + + for key in ("binary_sha256", "hostgate_sha256", "gate_sha256", "analyzer_sha256", "run_sha256"): + if HEX64.fullmatch(environment[key]) is None: + raise ValueError(f"{path}: {key} is not a 64-hex SHA-256") + for key in ("worktree", "binary", "mlx_checkout", "hostgate_source", "host", "uname", "gpu"): + if not environment[key].strip(): + raise ValueError(f"{path}: {key} must be nonempty") + + worktree = Path(environment["worktree"]) + try: + subprocess.run( + [ + "git", + "-C", + str(worktree), + "cat-file", + "-e", + f"{environment['commit']}^{{commit}}", + ], + check=True, + capture_output=True, + ) + except (OSError, subprocess.CalledProcessError) as exc: + raise ValueError(f"{path}: recorded commit is unavailable in the worktree") from exc + + binary = Path(environment["binary"]) + if not binary.is_file(): + raise ValueError(f"{path}: recorded binary is missing: {binary}") + if sha256_file(binary) != environment["binary_sha256"]: + raise ValueError(f"{path}: recorded binary SHA-256 does not match the artifact") + try: + recorded_size = int(environment["binary_size_bytes"]) + except ValueError as exc: + raise ValueError(f"{path}: binary_size_bytes is not an integer") from exc + if recorded_size != binary.stat().st_size: + raise ValueError(f"{path}: recorded binary size does not match the artifact") + binary_mtime = parse_utc(environment["binary_mtime_utc"], f"{path}:binary_mtime_utc") + actual_mtime = dt.datetime.fromtimestamp( + int(binary.stat().st_mtime), tz=dt.UTC + ) + if binary_mtime != actual_mtime: + raise ValueError(f"{path}: recorded binary mtime does not match the artifact") + if binary_mtime > started: + raise ValueError(f"{path}: binary was modified after the sweep started") + + provenance_files = { + "hostgate_sha256": Path(environment["hostgate_source"]), + "gate_sha256": data / "gate.py", + "analyzer_sha256": data / "analyze.py", + "run_sha256": data / "run.sh", + } + for key, artifact in provenance_files.items(): + if not artifact.is_file(): + raise ValueError(f"{path}: provenance artifact is missing: {artifact}") + if sha256_file(artifact) != environment[key]: + raise ValueError(f"{path}: {key} does not match {artifact}") + + return environment, started, completed + + +def read_schedule( + data: Path, + *, + run_started: dt.datetime, + run_completed: dt.datetime, +) -> dict[str, dt.datetime]: + """Validate the exact 60-row arm order and backend mapping.""" + + path = data / "schedule.tsv" + if not path.is_file(): + raise ValueError(f"missing schedule: {path}") + with path.open(newline="", encoding="utf-8") as source: + reader = csv.DictReader(source, delimiter="\t") + if tuple(reader.fieldnames or ()) != SCHEDULE_FIELDS: + raise ValueError(f"{path}: unexpected header {reader.fieldnames}") + rows = list(reader) + + expected = expected_arms() + if len(rows) != len(expected): + raise ValueError(f"{path}: expected {len(expected)} rows, found {len(rows)}") + + times: dict[str, dt.datetime] = {} + previous_started = run_started + for line_no, (row, spec) in enumerate(zip(rows, expected, strict=True), start=2): + try: + observed = ArmSpec( + model=row["model"], + prompt_tokens=int(row["prompt_tokens"]), + round=int(row["round"]), + position=int(row["position"]), + arm=row["arm"], + backend=row["backend"], + ) + except (TypeError, ValueError) as exc: + raise ValueError(f"{path}:{line_no}: invalid numeric field") from exc + if observed != spec: + raise ValueError(f"{path}:{line_no}: expected {spec}, found {observed}") + if spec.label in times: + raise ValueError(f"{path}:{line_no}: duplicate arm label {spec.label}") + started = parse_utc(row["started_utc"], f"{path}:{line_no}:started_utc") + if not (run_started <= started <= run_completed): + raise ValueError(f"{path}:{line_no}: arm timestamp is outside the run") + if started < previous_started: + raise ValueError(f"{path}:{line_no}: arm timestamps are not nondecreasing") + times[spec.label] = started + previous_started = started + return times + + +def require_bool(record: dict[str, Any], key: str, expected: bool, context: str) -> None: + """Require an exact JSON boolean.""" + + if record.get(key) is not expected: + raise ValueError(f"{context}: {key} must be {str(expected).lower()}") + + +def require_nonnegative_number(record: dict[str, Any], key: str, context: str) -> float: + """Require a finite nonnegative JSON number and return it.""" + + value = record.get(key) + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise ValueError(f"{context}: {key} must be numeric") + value = float(value) + if not math.isfinite(value) or value < 0: + raise ValueError(f"{context}: {key} must be finite and nonnegative") + return value + + +def read_hostgate( + data: Path, + *, + schedule_times: dict[str, dt.datetime], + run_started: dt.datetime, + run_completed: dt.datetime, +) -> None: + """Require one clean, accepted gate record for every arm plus preflight.""" + + path = data / "hostgate.jsonl" + if not path.is_file(): + raise ValueError(f"missing host-gate evidence: {path}") + records: list[dict[str, Any]] = [] + for line_no, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + raise ValueError(f"{path}:{line_no}: blank evidence line") + try: + record = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError(f"{path}:{line_no}: invalid JSON") from exc + if not isinstance(record, dict): + raise ValueError(f"{path}:{line_no}: evidence must be a JSON object") + records.append(record) + + expected_labels = ["run-preflight", *(spec.label for spec in expected_arms())] + labels = [record.get("label") for record in records] + if len(records) != len(expected_labels): + raise ValueError( + f"{path}: expected {len(expected_labels)} records, found {len(records)}" + ) + if labels != expected_labels: + raise ValueError(f"{path}: gate labels are missing, duplicated, or out of order") + + by_label = {str(record["label"]): record for record in records} + preflight = by_label["run-preflight"] + require_bool(preflight, "port_free_before_wait", True, f"{path}:run-preflight") + require_bool(preflight, "port_free_after_wait", True, f"{path}:run-preflight") + if preflight.get("port") != 18080: + raise ValueError(f"{path}:run-preflight: unexpected port") + preflight_started = parse_utc(preflight.get("started_utc"), f"{path}:run-preflight:started") + preflight_completed = parse_utc( + preflight.get("completed_utc"), f"{path}:run-preflight:completed" + ) + if not (run_started <= preflight_started <= preflight_completed <= run_completed): + raise ValueError(f"{path}:run-preflight: timestamps are outside the run") + + previous_completed = preflight_completed + for spec in expected_arms(): + context = f"{path}:{spec.label}" + record = by_label[spec.label] + if record.get("port") != 18080: + raise ValueError(f"{context}: unexpected port") + require_bool(record, "port_free_before_wait", True, context) + require_bool(record, "port_free_after_wait", True, context) + require_bool(record, "accepted_quiet", True, context) + require_bool(record, "ci_job_running", False, context) + if record.get("busy_procs") != []: + raise ValueError(f"{context}: busy_procs must be empty") + if record.get("foreign_model_procs") != []: + raise ValueError(f"{context}: foreign_model_procs must be empty") + for key in ( + "gate_wait_s", + "driver_wait_s", + "mem_wait_s", + "load1", + "mem_available_gib", + "mem_floor_gib", + "nvrm_window_5m", + "nvrm_window_max", + "nvrm_total_since_boot", + "nvrm_total_max", + "quiet_rechecks", + ): + require_nonnegative_number(record, key, context) + if float(record["mem_available_gib"]) < float(record["mem_floor_gib"]): + raise ValueError(f"{context}: memory evidence is below the gate floor") + if int(record["nvrm_window_5m"]) >= int(record["nvrm_window_max"]): + raise ValueError(f"{context}: NVRM window evidence exceeds the gate") + if int(record["nvrm_total_since_boot"]) >= int(record["nvrm_total_max"]): + raise ValueError(f"{context}: cumulative NVRM evidence exceeds the gate") + started = parse_utc(record.get("started_utc"), f"{context}:started") + completed = parse_utc(record.get("completed_utc"), f"{context}:completed") + if not (run_started <= started <= completed <= schedule_times[spec.label]): + raise ValueError(f"{context}: gate did not complete before its scheduled arm") + if started < previous_completed: + raise ValueError(f"{context}: gate timestamps are overlapping or out of order") + previous_completed = completed - return 100.0 * (value / baseline - 1.0) +def read_server_logs(data: Path) -> None: + """Require one nonempty server log with the requested backend per arm.""" -def read_runs() -> list[Run]: - """Read all expected client logs and reject incomplete or failed arms.""" + specs = expected_arms() + expected_names = {f"server-{spec.label}.log" for spec in specs} + actual_paths = {path.name: path for path in data.glob("server-*.log")} + if set(actual_paths) != expected_names: + missing = sorted(expected_names - set(actual_paths)) + extra = sorted(set(actual_paths) - expected_names) + raise ValueError(f"server log set mismatch; missing={missing}, extra={extra}") + + for spec in specs: + path = actual_paths[f"server-{spec.label}.log"] + if path.stat().st_size == 0: + raise ValueError(f"{path}: server log is empty") + with path.open(encoding="utf-8", errors="replace") as source: + marker = source.readline().rstrip("\n") + expected_marker = f"requested_decode_storage_backend={spec.backend}" + if marker != expected_marker: + raise ValueError(f"{path}: expected backend marker {expected_marker!r}") + + +def read_runs(data: Path) -> tuple[Run, ...]: + """Read the exact client-log set and reject duplicate, failed, or missing rows.""" + + specs = expected_arms() + expected_names = {spec.client_name for spec in specs} + actual_paths = {path.name: path for path in data.glob("client-*.txt")} + if set(actual_paths) != expected_names: + missing = sorted(expected_names - set(actual_paths)) + extra = sorted(set(actual_paths) - expected_names) + raise ValueError(f"client log set mismatch; missing={missing}, extra={extra}") runs: list[Run] = [] - for path in sorted(DATA.glob("client-*.txt")): + keys: set[tuple[str, int, int, str, int]] = set() + for spec in specs: + path = actual_paths[spec.client_name] match = LOG_NAME.fullmatch(path.name) if match is None: - continue - rows = [] - for line in path.read_text().splitlines(): + raise ValueError(f"unexpected client log name: {path.name}") + rows: dict[int, Run] = {} + for line in path.read_text(encoding="utf-8").splitlines(): if row := TABLE_ROW.match(line): + concurrency = int(row["concurrency"]) + if concurrency in rows: + raise ValueError(f"{path}: duplicate concurrency {concurrency}") if int(row["failed"]) != 0: - raise ValueError(f"{path}: concurrency {row['concurrency']} has failures") - rows.append( - Run( - model=match["model"], - prompt_tokens=int(match["prompt"]), - round=int(match["round"]), - arm=match["arm"], - concurrency=int(row["concurrency"]), - aggregate_tok_s=float(row["aggregate"]), - ) + raise ValueError(f"{path}: concurrency {concurrency} has failures") + aggregate = float(row["aggregate"]) + if not math.isfinite(aggregate) or aggregate <= 0: + raise ValueError(f"{path}: concurrency {concurrency} has invalid throughput") + rows[concurrency] = Run( + model=match["model"], + prompt_tokens=int(match["prompt"]), + round=int(match["round"]), + arm=match["arm"], + concurrency=concurrency, + aggregate_tok_s=aggregate, ) - if len(rows) != 3: - raise ValueError(f"{path}: expected three concurrency rows, found {len(rows)}") - runs.extend(rows) + if set(rows) != set(CONCURRENCIES): + raise ValueError( + f"{path}: expected concurrency rows {CONCURRENCIES}, found {sorted(rows)}" + ) + for concurrency in CONCURRENCIES: + run = rows[concurrency] + key = (run.model, run.prompt_tokens, run.round, run.arm, run.concurrency) + if key in keys: + raise ValueError(f"{path}: duplicate measurement key {key}") + keys.add(key) + runs.append(run) - expected = 2 * 2 * 5 * 3 * 3 - if len(runs) != expected: - raise ValueError(f"expected {expected} measurements, found {len(runs)}") - return runs + expected_keys = { + (spec.model, spec.prompt_tokens, spec.round, spec.arm, concurrency) + for spec in specs + for concurrency in CONCURRENCIES + } + if keys != expected_keys: + raise ValueError("client measurement keys do not match the 180-row protocol") + return tuple(runs) + + +def validate_protocol(data: Path) -> Protocol: + """Validate every acceptance artifact before allowing any calculation.""" + if not (data / "complete").is_file(): + raise ValueError(f"missing completion marker: {data / 'complete'}") + environment, run_started, run_completed = read_environment(data) + schedule_times = read_schedule( + data, + run_started=run_started, + run_completed=run_completed, + ) + read_hostgate( + data, + schedule_times=schedule_times, + run_started=run_started, + run_completed=run_completed, + ) + read_server_logs(data) + runs = read_runs(data) + return Protocol(environment, schedule_times, runs) -def main() -> int: - """Print per-cell medians, paired deltas, null spreads, and family decisions.""" + +def percent_delta(value: float, baseline: float) -> float: + """Return the percentage delta of ``value`` from ``baseline``.""" + + return 100.0 * (value / baseline - 1.0) + + +def summarize(protocol: Protocol) -> None: + """Print per-cell medians, paired deltas, null spreads, and decisions.""" by_key = { (run.model, run.prompt_tokens, run.concurrency, run.round, run.arm): run.aggregate_tok_s - for run in read_runs() + for run in protocol.runs } decisions: dict[str, tuple[bool, bool]] = {} print("| Model | Prompt | C | paged-a median | dense median | paged-b median | dense paired delta | null spread | Cell |") print("|---|---:|---:|---:|---:|---:|---:|---:|---|") - for model in ("gemma3", "llama4"): + for model in MODELS: any_dense_win = False any_paged_win = False - for prompt in (512, 4096): - for concurrency in (1, 4, 8): + for prompt in PROMPTS: + for concurrency in CONCURRENCIES: arm_values = { arm: [ by_key[(model, prompt, concurrency, round_no, arm)] - for round_no in range(1, 6) + for round_no in ROUNDS ] - for arm in ("paged-a", "dense", "paged-b") + for arm in ARMS } - paired = [] - null = [] - for index in range(5): - paired.append( - percent_delta( - arm_values["dense"][index], arm_values["paged-a"][index] - ) + paired = [ + percent_delta(dense, paged) + for dense, paged in zip( + arm_values["dense"], arm_values["paged-a"], strict=True ) - null.append( - percent_delta( - arm_values["paged-b"][index], arm_values["paged-a"][index] - ) + ] + null = [ + percent_delta(paged_b, paged_a) + for paged_b, paged_a in zip( + arm_values["paged-b"], arm_values["paged-a"], strict=True ) + ] paired_median = statistics.median(paired) null_low, null_high = min(null), max(null) @@ -133,6 +585,15 @@ def main() -> int: f"{model}: supports_paged_decode_backend={'false' if use_dense else 'true'} " f"(dense_clear_cell={any_dense_win}, paged_clear_cell={any_paged_win})" ) + + +def main(argv: list[str] | None = None) -> int: + """Validate the sweep, then print its statistical summary.""" + + parser = argparse.ArgumentParser() + parser.add_argument("--data-dir", type=Path, default=DATA) + args = parser.parse_args(argv) + summarize(validate_protocol(args.data_dir.resolve())) return 0 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/gate.py b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/gate.py index 21ab65e50..ea0bfb239 100755 --- a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/gate.py +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/gate.py @@ -11,6 +11,7 @@ import socket import sys from pathlib import Path +from typing import Any def assert_port_free(port: int) -> None: @@ -29,6 +30,77 @@ def utc_now() -> str: return dt.datetime.now(dt.UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z") +def quiet_snapshot(gate: Any) -> dict[str, object]: + """Capture every host condition that can invalidate a timed arm.""" + + nvrm_window, nvrm_total = gate.nvrm_counts() + return { + "ci_job_running": gate.ci_job_running(), + "busy_procs": gate.busy_procs(), + "foreign_model_procs": gate.foreign_model_procs(), + "load1": os.getloadavg()[0], + "mem_available_gib": gate.mem_available_gib(), + "mem_floor_gib": gate.MEM_FLOOR_GIB, + "nvrm_window_5m": nvrm_window, + "nvrm_window_max": gate.NVRM_WINDOW_MAX, + "nvrm_total_since_boot": nvrm_total, + "nvrm_total_max": gate.NVRM_TOTAL_MAX, + } + + +def snapshot_is_quiet(snapshot: dict[str, object]) -> bool: + """Return whether a final post-wait snapshot satisfies every host gate.""" + + return ( + snapshot["ci_job_running"] is False + and snapshot["busy_procs"] == [] + and snapshot["foreign_model_procs"] == [] + and float(snapshot["mem_available_gib"]) >= float(snapshot["mem_floor_gib"]) + and int(snapshot["nvrm_window_5m"]) < int(snapshot["nvrm_window_max"]) + and int(snapshot["nvrm_total_since_boot"]) < int(snapshot["nvrm_total_max"]) + ) + + +def wait_for_accepted_quiet( + gate: Any, + *, + max_post_wait_rechecks: int, +) -> dict[str, object]: + """Wait for quiet and retry when activity appears in the post-wait race.""" + + quiet_wait_s = 0 + driver_wait_s = 0 + mem_wait_s = 0 + for recheck in range(max_post_wait_rechecks + 1): + quiet_wait_s += gate.wait_quiet(log=sys.stderr) + waited, _window, _total = gate.driver_gate(log=sys.stderr) + driver_wait_s += waited + waited, _available = gate.mem_gate(log=sys.stderr) + mem_wait_s += waited + snapshot = quiet_snapshot(gate) + if snapshot_is_quiet(snapshot): + snapshot.update( + accepted_quiet=True, + quiet_rechecks=recheck, + gate_wait_s=quiet_wait_s, + driver_wait_s=driver_wait_s, + mem_wait_s=mem_wait_s, + ) + return snapshot + print( + "[gate] activity appeared after the sustained wait; retrying: " + f"ci={snapshot['ci_job_running']} busy={snapshot['busy_procs']} " + f"foreign={snapshot['foreign_model_procs']} " + f"mem={float(snapshot['mem_available_gib']):.1f}GiB " + f"nvrm={snapshot['nvrm_window_5m']}/{snapshot['nvrm_total_since_boot']}", + file=sys.stderr, + flush=True, + ) + raise RuntimeError( + "host became nonquiet after every sustained wait; refusing to start a timed arm" + ) + + def main() -> int: """Gate one arm, fail closed on port conflicts, and append its evidence.""" @@ -38,8 +110,12 @@ def main() -> int: parser.add_argument("--label", required=True) parser.add_argument("--port", type=int, required=True) parser.add_argument("--port-only", action="store_true") + parser.add_argument("--max-post-wait-rechecks", type=int, default=12) args = parser.parse_args() + if args.max_post_wait_rechecks < 0: + parser.error("--max-post-wait-rechecks must be non-negative") + started = utc_now() assert_port_free(args.port) record: dict[str, object] = { @@ -49,21 +125,20 @@ def main() -> int: "port_free_before_wait": True, } - if not args.port_only: + if args.port_only: + assert_port_free(args.port) + record["port_free_after_wait"] = True + else: sys.path.insert(0, str(args.hostgate_dir)) gate = importlib.import_module("hostgate") - record["gate_wait_s"] = gate.wait_quiet(log=sys.stderr) - assert_port_free(args.port) record.update( - port_free_after_wait=True, - ci_job_running=gate.ci_job_running(), - busy_procs=gate.busy_procs(), - foreign_model_procs=gate.foreign_model_procs(), - load1=os.getloadavg()[0], - mem_available_gib=gate.mem_available_gib(), + wait_for_accepted_quiet( + gate, + max_post_wait_rechecks=args.max_post_wait_rechecks, + ) ) - nvrm_window, nvrm_total = gate.nvrm_counts() - record.update(nvrm_window_5m=nvrm_window, nvrm_total_since_boot=nvrm_total) + assert_port_free(args.port) + record["port_free_after_wait"] = True record["completed_utc"] = utc_now() args.evidence.parent.mkdir(parents=True, exist_ok=True) diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/run.sh b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/run.sh index 08ee101db..756a9c26b 100755 --- a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/run.sh +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/run.sh @@ -7,6 +7,7 @@ CLIENT=/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/sc MODEL_ROOT=/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/models/mlx DATA=/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11 GATE_DRIVER=${DATA}/gate.py +ANALYZER=${DATA}/analyze.py HOSTGATE_DIR=${WORKTREE}/docs/benchmark_results/data/sdpa-plan-bucket-gb10-2026-09-12/harness PORT=18080 @@ -24,10 +25,7 @@ cleanup_server() { } trap cleanup_server EXIT INT TERM -mkdir -p "${DATA}" -exec > >(tee -a "${DATA}/run.log") 2>&1 - -for required in "${BIN}" "${CLIENT}" "${GATE_DRIVER}" "${HOSTGATE_DIR}/hostgate.py" "${MODEL_ROOT}/gemma-3-1b-it-4bit" "${MODEL_ROOT}/llama-4-scout-17b-16e-instruct-4bit" "${MLXCEL_CCCL_DIR}" "${MLXCEL_CUTLASS_DIR}" "${MLX_SOURCE_DIR}/.git"; do +for required in "${BIN}" "${CLIENT}" "${GATE_DRIVER}" "${ANALYZER}" "${HOSTGATE_DIR}/hostgate.py" "${MODEL_ROOT}/gemma-3-1b-it-4bit" "${MODEL_ROOT}/llama-4-scout-17b-16e-instruct-4bit" "${MLXCEL_CCCL_DIR}" "${MLXCEL_CUTLASS_DIR}" "${MLX_SOURCE_DIR}/.git"; do if [[ ! -e "${required}" ]]; then echo "missing required benchmark path: ${required}" >&2 exit 2 @@ -39,8 +37,15 @@ if [[ -e "${DATA}/complete" ]]; then exit 2 fi -: > "${DATA}/hostgate.jsonl" -python3 "${GATE_DRIVER}" --hostgate-dir "${HOSTGATE_DIR}" --evidence "${DATA}/hostgate.jsonl" --label run-preflight --port "${PORT}" --port-only +run_commit=$(git -C "${WORKTREE}" rev-parse HEAD) +if [[ -n "$(git -C "${WORKTREE}" status --porcelain --untracked-files=all)" ]]; then + echo "benchmark worktree must be clean before provenance is recorded" >&2 + git -C "${WORKTREE}" status --short >&2 + exit 2 +fi + +mkdir -p "${DATA}" +exec > >(tee -a "${DATA}/run.log") 2>&1 mlx_cmake_tag=$(awk '$1 == "GIT_TAG" { gsub(/[")]/, "", $2); print $2; exit }' "${WORKTREE}/src/lib/mlx-cpp/CMakeLists.txt") mlx_checkout_commit=$(git -C "${MLX_SOURCE_DIR}" rev-parse HEAD) @@ -52,19 +57,28 @@ fi { echo "started_utc=$(date -u +%FT%TZ)" echo "worktree=${WORKTREE}" + echo "worktree_clean=true" echo "binary=${BIN}" echo "binary_sha256=$(sha256sum "${BIN}" | awk '{print $1}')" - echo "commit=$(git -C "${WORKTREE}" rev-parse HEAD)" + echo "binary_size_bytes=$(stat -c %s "${BIN}")" + echo "binary_mtime_utc=$(date -u -d "@$(stat -c %Y "${BIN}")" +%FT%TZ)" + echo "commit=${run_commit}" echo "mlx_cmake_git_tag=${mlx_cmake_tag}" echo "mlx_checkout=${MLX_SOURCE_DIR}" echo "mlx_checkout_commit=${mlx_checkout_commit}" echo "hostgate_source=${HOSTGATE_DIR}/hostgate.py" echo "hostgate_sha256=$(sha256sum "${HOSTGATE_DIR}/hostgate.py" | awk '{print $1}')" + echo "gate_sha256=$(sha256sum "${GATE_DRIVER}" | awk '{print $1}')" + echo "analyzer_sha256=$(sha256sum "${ANALYZER}" | awk '{print $1}')" + echo "run_sha256=$(sha256sum "${DATA}/run.sh" | awk '{print $1}')" echo "host=$(hostname)" - uname -a - nvidia-smi --query-gpu=name,driver_version,pci.bus_id --format=csv,noheader + echo "uname=$(uname -a)" + echo "gpu=$(nvidia-smi --query-gpu=name,driver_version,pci.bus_id --format=csv,noheader | paste -sd ';' -)" } > "${DATA}/environment.txt" +: > "${DATA}/hostgate.jsonl" +python3 "${GATE_DRIVER}" --hostgate-dir "${HOSTGATE_DIR}" --evidence "${DATA}/hostgate.jsonl" --label run-preflight --port "${PORT}" --port-only + : > "${DATA}/schedule.tsv" printf 'model\tprompt_tokens\tround\tposition\tarm\tbackend\tstarted_utc\n' >> "${DATA}/schedule.tsv" @@ -102,7 +116,8 @@ run_arm() { python3 "${GATE_DRIVER}" --hostgate-dir "${HOSTGATE_DIR}" --evidence "${DATA}/hostgate.jsonl" --label "${stem}" --port "${PORT}" printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "${model_name}" "${prompt_tokens}" "${round}" "${position}" "${arm}" "${backend}" "$(date -u +%FT%TZ)" >> "${DATA}/schedule.tsv" echo "START ${stem} $(date -u +%FT%TZ)" - "${BIN}" serve -m "${model_path}" --port "${PORT}" --metrics --parallel 8 --decode-storage-backend "${backend}" > "${server_log}" 2>&1 & + printf 'requested_decode_storage_backend=%s\n' "${backend}" > "${server_log}" + "${BIN}" serve -m "${model_path}" --port "${PORT}" --metrics --parallel 8 --decode-storage-backend "${backend}" >> "${server_log}" 2>&1 & SERVER_PID=$! if ! wait_for_server; then diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/test_protocol.py b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/test_protocol.py new file mode 100755 index 000000000..484821f98 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/test_protocol.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 +"""Lightweight protocol fixtures for the issue #2230 storage sweep.""" + +from __future__ import annotations + +import datetime as dt +import hashlib +import importlib.util +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from types import ModuleType + + +HERE = Path(__file__).resolve().parent + + +def load_module(name: str, path: Path) -> ModuleType: + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +analyze = load_module("storage_analyze", HERE / "analyze.py") +gate_driver = load_module("storage_gate", HERE / "gate.py") + + +def file_sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +class ProtocolFixture: + """A complete synthetic 60-arm protocol with no model or GPU work.""" + + def __init__(self, root: Path) -> None: + self.root = root + self.worktree = Path( + subprocess.run( + ["git", "-C", str(HERE), "rev-parse", "--show-toplevel"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + ) + self.commit = subprocess.run( + ["git", "-C", str(self.worktree), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + def write(self) -> None: + self.root.mkdir(parents=True, exist_ok=True) + for name in ("gate.py", "analyze.py", "run.sh"): + (self.root / name).write_text(f"fixture {name}\n", encoding="utf-8") + hostgate = self.root / "hostgate.py" + hostgate.write_text("fixture hostgate\n", encoding="utf-8") + binary = self.root / "mlxcel" + binary.write_bytes(b"fixture release binary") + binary_epoch = int(dt.datetime(2026, 10, 10, 23, tzinfo=dt.UTC).timestamp()) + os.utime(binary, (binary_epoch, binary_epoch)) + + environment = { + "started_utc": "2026-10-11T00:00:00Z", + "completed_utc": "2026-10-11T23:00:00Z", + "worktree": str(self.worktree), + "worktree_clean": "true", + "binary": str(binary), + "binary_sha256": file_sha256(binary), + "binary_size_bytes": str(binary.stat().st_size), + "binary_mtime_utc": "2026-10-10T23:00:00Z", + "commit": self.commit, + "mlx_cmake_git_tag": "1" * 40, + "mlx_checkout": str(self.root / "mlx-src"), + "mlx_checkout_commit": "1" * 40, + "hostgate_source": str(hostgate), + "hostgate_sha256": file_sha256(hostgate), + "gate_sha256": file_sha256(self.root / "gate.py"), + "analyzer_sha256": file_sha256(self.root / "analyze.py"), + "run_sha256": file_sha256(self.root / "run.sh"), + "host": "fixture-host", + "uname": "Linux fixture", + "gpu": "Fixture GPU, 1.0, 0000:00:00.0", + } + (self.root / "environment.txt").write_text( + "".join(f"{key}={value}\n" for key, value in environment.items()), + encoding="utf-8", + ) + (self.root / "complete").touch() + + schedule_lines = ["\t".join(analyze.SCHEDULE_FIELDS)] + evidence = [ + { + "label": "run-preflight", + "started_utc": "2026-10-11T00:00:01Z", + "completed_utc": "2026-10-11T00:00:02Z", + "port": 18080, + "port_free_before_wait": True, + "port_free_after_wait": True, + } + ] + for index, spec in enumerate(analyze.expected_arms(), start=1): + scheduled = dt.datetime(2026, 10, 11, tzinfo=dt.UTC) + dt.timedelta( + minutes=10 * index + ) + gate_started = scheduled - dt.timedelta(minutes=2) + gate_completed = scheduled - dt.timedelta(minutes=1) + scheduled_utc = scheduled.strftime("%Y-%m-%dT%H:%M:%SZ") + gate_started_utc = gate_started.strftime("%Y-%m-%dT%H:%M:%SZ") + gate_completed_utc = gate_completed.strftime("%Y-%m-%dT%H:%M:%SZ") + schedule_lines.append( + "\t".join( + ( + spec.model, + str(spec.prompt_tokens), + str(spec.round), + str(spec.position), + spec.arm, + spec.backend, + scheduled_utc, + ) + ) + ) + evidence.append( + { + "label": spec.label, + "started_utc": gate_started_utc, + "completed_utc": gate_completed_utc, + "port": 18080, + "port_free_before_wait": True, + "port_free_after_wait": True, + "accepted_quiet": True, + "ci_job_running": False, + "busy_procs": [], + "foreign_model_procs": [], + "gate_wait_s": 0, + "driver_wait_s": 0, + "mem_wait_s": 0, + "load1": 0.5, + "mem_available_gib": 64.0, + "mem_floor_gib": 32.0, + "nvrm_window_5m": 0, + "nvrm_window_max": 50, + "nvrm_total_since_boot": 0, + "nvrm_total_max": 400, + "quiet_rechecks": index % 2, + } + ) + (self.root / f"server-{spec.label}.log").write_text( + f"requested_decode_storage_backend={spec.backend}\nfixture server output\n", + encoding="utf-8", + ) + (self.root / spec.client_name).write_text( + "".join( + f"{concurrency} 1 0 1.0 2.0 3.0 {100.0 + index + concurrency:.1f}\n" + for concurrency in analyze.CONCURRENCIES + ), + encoding="utf-8", + ) + (self.root / "schedule.tsv").write_text( + "\n".join(schedule_lines) + "\n", encoding="utf-8" + ) + (self.root / "hostgate.jsonl").write_text( + "".join(json.dumps(record, sort_keys=True) + "\n" for record in evidence), + encoding="utf-8", + ) + + +class ProtocolValidationTests(unittest.TestCase): + def test_complete_quiet_protocol_is_accepted(self) -> None: + with tempfile.TemporaryDirectory() as temp: + fixture = ProtocolFixture(Path(temp)) + fixture.write() + protocol = analyze.validate_protocol(fixture.root) + self.assertEqual(len(protocol.schedule_times), 60) + self.assertEqual(len(protocol.runs), 180) + + def test_missing_gate_evidence_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as temp: + fixture = ProtocolFixture(Path(temp)) + fixture.write() + path = fixture.root / "hostgate.jsonl" + lines = path.read_text(encoding="utf-8").splitlines() + path.write_text("\n".join(lines[:-1]) + "\n", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "expected 61 records"): + analyze.validate_protocol(fixture.root) + + def test_dirty_gate_evidence_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as temp: + fixture = ProtocolFixture(Path(temp)) + fixture.write() + path = fixture.root / "hostgate.jsonl" + records = [json.loads(line) for line in path.read_text().splitlines()] + records[1]["busy_procs"] = ["rustc"] + path.write_text( + "".join(json.dumps(record) + "\n" for record in records), + encoding="utf-8", + ) + with self.assertRaisesRegex(ValueError, "busy_procs must be empty"): + analyze.validate_protocol(fixture.root) + + def test_reordered_gate_evidence_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as temp: + fixture = ProtocolFixture(Path(temp)) + fixture.write() + path = fixture.root / "hostgate.jsonl" + records = path.read_text(encoding="utf-8").splitlines() + records[1], records[2] = records[2], records[1] + path.write_text("\n".join(records) + "\n", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "out of order"): + analyze.validate_protocol(fixture.root) + + def test_nonmonotonic_schedule_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as temp: + fixture = ProtocolFixture(Path(temp)) + fixture.write() + path = fixture.root / "schedule.tsv" + rows = path.read_text(encoding="utf-8").splitlines() + fields = rows[2].split("\t") + fields[-1] = "2026-10-11T00:05:00Z" + rows[2] = "\t".join(fields) + path.write_text("\n".join(rows) + "\n", encoding="utf-8") + with self.assertRaisesRegex(ValueError, "not nondecreasing"): + analyze.validate_protocol(fixture.root) + + def test_missing_server_log_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as temp: + fixture = ProtocolFixture(Path(temp)) + fixture.write() + first = analyze.expected_arms()[0] + (fixture.root / f"server-{first.label}.log").unlink() + with self.assertRaisesRegex(ValueError, "server log set mismatch"): + analyze.validate_protocol(fixture.root) + + def test_wrong_server_backend_marker_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as temp: + fixture = ProtocolFixture(Path(temp)) + fixture.write() + first = analyze.expected_arms()[0] + path = fixture.root / f"server-{first.label}.log" + path.write_text( + "requested_decode_storage_backend=dense\nfixture server output\n", + encoding="utf-8", + ) + with self.assertRaisesRegex(ValueError, "expected backend marker"): + analyze.validate_protocol(fixture.root) + + +class FakeHostgate: + NVRM_WINDOW_MAX = 50 + NVRM_TOTAL_MAX = 400 + MEM_FLOOR_GIB = 32.0 + + def __init__(self, *, always_dirty: bool = False) -> None: + self.attempt = -1 + self.always_dirty = always_dirty + + def wait_quiet(self, *, log: object) -> int: + del log + self.attempt += 1 + return 0 + + def driver_gate(self, *, log: object) -> tuple[int, int, int]: + del log + return 0, 0, 0 + + def mem_gate(self, *, log: object) -> tuple[int, float]: + del log + return 0, 64.0 + + def ci_job_running(self) -> bool: + return False + + def busy_procs(self) -> list[str]: + return ["rustc"] if self.always_dirty or self.attempt == 0 else [] + + def foreign_model_procs(self) -> list[str]: + return [] + + def mem_available_gib(self) -> float: + return 64.0 + + def nvrm_counts(self) -> tuple[int, int]: + return 0, 0 + + +class QuietGateTests(unittest.TestCase): + def test_post_wait_race_retries_until_clean(self) -> None: + fake = FakeHostgate() + record = gate_driver.wait_for_accepted_quiet(fake, max_post_wait_rechecks=2) + self.assertTrue(record["accepted_quiet"]) + self.assertEqual(record["quiet_rechecks"], 1) + self.assertEqual(fake.attempt, 1) + + def test_repeated_dirty_snapshots_fail_closed(self) -> None: + fake = FakeHostgate(always_dirty=True) + with self.assertRaisesRegex(RuntimeError, "refusing to start"): + gate_driver.wait_for_accepted_quiet(fake, max_post_wait_rechecks=1) + + +if __name__ == "__main__": + unittest.main() From 279208de4c12c7d1fd0a6399e9fdee36c5aac1e5 Mon Sep 17 00:00:00 2001 From: Jeongkyu Shin Date: Sun, 11 Oct 2026 16:21:41 +0900 Subject: [PATCH 4/7] docs: record storage and SSM evidence Record the complete 60-arm paired/null storage sweep and retain paged storage for Gemma 3 and Llama 4 because no cell clears its repeated-paged null spread. Preserve the raw host-gate, client, server, schedule, artifact, derived binary-source provenance, SSM stage, and Granite checkpoint evidence with checksums. Cite the dated decision from both model flags and the continuous-batching guide. The original measured scripts and their recorded hashes remain unchanged. Validation: analyzer accepted all 60 arms and 180 cells; nine protocol fixtures passed; both artifact manifests verified. Refs #2230 --- docs/CONTINUOUS_BATCHING.md | 8 + .../README.md | 45 +++++ .../SHA256SUMS | 132 +++++++++++++ .../analysis.txt | 17 ++ .../binary-source-provenance.json | 26 +++ .../client-gemma3-p4096-r1-dense.txt | 30 +++ .../client-gemma3-p4096-r1-paged-a.txt | 30 +++ .../client-gemma3-p4096-r1-paged-b.txt | 30 +++ .../client-gemma3-p4096-r2-dense.txt | 30 +++ .../client-gemma3-p4096-r2-paged-a.txt | 30 +++ .../client-gemma3-p4096-r2-paged-b.txt | 30 +++ .../client-gemma3-p4096-r3-dense.txt | 30 +++ .../client-gemma3-p4096-r3-paged-a.txt | 30 +++ .../client-gemma3-p4096-r3-paged-b.txt | 30 +++ .../client-gemma3-p4096-r4-dense.txt | 30 +++ .../client-gemma3-p4096-r4-paged-a.txt | 30 +++ .../client-gemma3-p4096-r4-paged-b.txt | 30 +++ .../client-gemma3-p4096-r5-dense.txt | 30 +++ .../client-gemma3-p4096-r5-paged-a.txt | 30 +++ .../client-gemma3-p4096-r5-paged-b.txt | 30 +++ .../client-gemma3-p512-r1-dense.txt | 30 +++ .../client-gemma3-p512-r1-paged-a.txt | 30 +++ .../client-gemma3-p512-r1-paged-b.txt | 30 +++ .../client-gemma3-p512-r2-dense.txt | 30 +++ .../client-gemma3-p512-r2-paged-a.txt | 30 +++ .../client-gemma3-p512-r2-paged-b.txt | 30 +++ .../client-gemma3-p512-r3-dense.txt | 30 +++ .../client-gemma3-p512-r3-paged-a.txt | 30 +++ .../client-gemma3-p512-r3-paged-b.txt | 30 +++ .../client-gemma3-p512-r4-dense.txt | 30 +++ .../client-gemma3-p512-r4-paged-a.txt | 30 +++ .../client-gemma3-p512-r4-paged-b.txt | 30 +++ .../client-gemma3-p512-r5-dense.txt | 30 +++ .../client-gemma3-p512-r5-paged-a.txt | 30 +++ .../client-gemma3-p512-r5-paged-b.txt | 30 +++ .../client-llama4-p4096-r1-dense.txt | 30 +++ .../client-llama4-p4096-r1-paged-a.txt | 30 +++ .../client-llama4-p4096-r1-paged-b.txt | 30 +++ .../client-llama4-p4096-r2-dense.txt | 30 +++ .../client-llama4-p4096-r2-paged-a.txt | 30 +++ .../client-llama4-p4096-r2-paged-b.txt | 30 +++ .../client-llama4-p4096-r3-dense.txt | 30 +++ .../client-llama4-p4096-r3-paged-a.txt | 30 +++ .../client-llama4-p4096-r3-paged-b.txt | 30 +++ .../client-llama4-p4096-r4-dense.txt | 30 +++ .../client-llama4-p4096-r4-paged-a.txt | 30 +++ .../client-llama4-p4096-r4-paged-b.txt | 30 +++ .../client-llama4-p4096-r5-dense.txt | 30 +++ .../client-llama4-p4096-r5-paged-a.txt | 30 +++ .../client-llama4-p4096-r5-paged-b.txt | 30 +++ .../client-llama4-p512-r1-dense.txt | 30 +++ .../client-llama4-p512-r1-paged-a.txt | 30 +++ .../client-llama4-p512-r1-paged-b.txt | 30 +++ .../client-llama4-p512-r2-dense.txt | 30 +++ .../client-llama4-p512-r2-paged-a.txt | 30 +++ .../client-llama4-p512-r2-paged-b.txt | 30 +++ .../client-llama4-p512-r3-dense.txt | 30 +++ .../client-llama4-p512-r3-paged-a.txt | 30 +++ .../client-llama4-p512-r3-paged-b.txt | 30 +++ .../client-llama4-p512-r4-dense.txt | 30 +++ .../client-llama4-p512-r4-paged-a.txt | 30 +++ .../client-llama4-p512-r4-paged-b.txt | 30 +++ .../client-llama4-p512-r5-dense.txt | 30 +++ .../client-llama4-p512-r5-paged-a.txt | 30 +++ .../client-llama4-p512-r5-paged-b.txt | 30 +++ .../complete | 0 .../environment.txt | 20 ++ .../hostgate.jsonl | 61 ++++++ .../run.log | 181 ++++++++++++++++++ .../schedule.tsv | 61 ++++++ .../server-gemma3-p4096-r1-dense.log | 48 +++++ .../server-gemma3-p4096-r1-paged-a.log | 48 +++++ .../server-gemma3-p4096-r1-paged-b.log | 48 +++++ .../server-gemma3-p4096-r2-dense.log | 48 +++++ .../server-gemma3-p4096-r2-paged-a.log | 48 +++++ .../server-gemma3-p4096-r2-paged-b.log | 48 +++++ .../server-gemma3-p4096-r3-dense.log | 48 +++++ .../server-gemma3-p4096-r3-paged-a.log | 48 +++++ .../server-gemma3-p4096-r3-paged-b.log | 48 +++++ .../server-gemma3-p4096-r4-dense.log | 48 +++++ .../server-gemma3-p4096-r4-paged-a.log | 48 +++++ .../server-gemma3-p4096-r4-paged-b.log | 48 +++++ .../server-gemma3-p4096-r5-dense.log | 48 +++++ .../server-gemma3-p4096-r5-paged-a.log | 48 +++++ .../server-gemma3-p4096-r5-paged-b.log | 48 +++++ .../server-gemma3-p512-r1-dense.log | 48 +++++ .../server-gemma3-p512-r1-paged-a.log | 48 +++++ .../server-gemma3-p512-r1-paged-b.log | 48 +++++ .../server-gemma3-p512-r2-dense.log | 48 +++++ .../server-gemma3-p512-r2-paged-a.log | 48 +++++ .../server-gemma3-p512-r2-paged-b.log | 48 +++++ .../server-gemma3-p512-r3-dense.log | 48 +++++ .../server-gemma3-p512-r3-paged-a.log | 48 +++++ .../server-gemma3-p512-r3-paged-b.log | 48 +++++ .../server-gemma3-p512-r4-dense.log | 48 +++++ .../server-gemma3-p512-r4-paged-a.log | 48 +++++ .../server-gemma3-p512-r4-paged-b.log | 48 +++++ .../server-gemma3-p512-r5-dense.log | 48 +++++ .../server-gemma3-p512-r5-paged-a.log | 48 +++++ .../server-gemma3-p512-r5-paged-b.log | 48 +++++ .../server-llama4-p4096-r1-dense.log | 50 +++++ .../server-llama4-p4096-r1-paged-a.log | 50 +++++ .../server-llama4-p4096-r1-paged-b.log | 50 +++++ .../server-llama4-p4096-r2-dense.log | 50 +++++ .../server-llama4-p4096-r2-paged-a.log | 50 +++++ .../server-llama4-p4096-r2-paged-b.log | 50 +++++ .../server-llama4-p4096-r3-dense.log | 50 +++++ .../server-llama4-p4096-r3-paged-a.log | 50 +++++ .../server-llama4-p4096-r3-paged-b.log | 50 +++++ .../server-llama4-p4096-r4-dense.log | 50 +++++ .../server-llama4-p4096-r4-paged-a.log | 50 +++++ .../server-llama4-p4096-r4-paged-b.log | 50 +++++ .../server-llama4-p4096-r5-dense.log | 50 +++++ .../server-llama4-p4096-r5-paged-a.log | 50 +++++ .../server-llama4-p4096-r5-paged-b.log | 50 +++++ .../server-llama4-p512-r1-dense.log | 50 +++++ .../server-llama4-p512-r1-paged-a.log | 51 +++++ .../server-llama4-p512-r1-paged-b.log | 50 +++++ .../server-llama4-p512-r2-dense.log | 50 +++++ .../server-llama4-p512-r2-paged-a.log | 50 +++++ .../server-llama4-p512-r2-paged-b.log | 50 +++++ .../server-llama4-p512-r3-dense.log | 50 +++++ .../server-llama4-p512-r3-paged-a.log | 50 +++++ .../server-llama4-p512-r3-paged-b.log | 50 +++++ .../server-llama4-p512-r4-dense.log | 50 +++++ .../server-llama4-p512-r4-paged-a.log | 50 +++++ .../server-llama4-p512-r4-paged-b.log | 50 +++++ .../server-llama4-p512-r5-dense.log | 50 +++++ .../server-llama4-p512-r5-paged-a.log | 50 +++++ .../server-llama4-p512-r5-paged-b.log | 50 +++++ .../ssm-bf16-parity-gb10-2026-10-11/README.md | 29 +++ .../SHA256SUMS | 9 + .../granite-parity/comparison.json | 139 ++++++++++++++ .../granite-parity/graph.stderr | 2 + .../granite-parity/graph.stdout | 12 ++ .../granite-parity/kernel.stderr | 3 + .../granite-parity/kernel.stdout | 12 ++ .../granite-parity/provenance.txt | 5 + .../granite-parity/run.sh | 136 +++++++++++++ .../stage-deviations.log | 39 ++++ .../model-owned-storage-gb10-2026-10-11.md | 75 ++++++++ src/models/gemma3.rs | 4 + src/models/llama4.rs | 4 + 143 files changed, 5761 insertions(+) create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/README.md create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/SHA256SUMS create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/analysis.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/binary-source-provenance.json create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r1-dense.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r1-paged-a.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r1-paged-b.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r2-dense.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r2-paged-a.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r2-paged-b.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r3-dense.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r3-paged-a.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r3-paged-b.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r4-dense.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r4-paged-a.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r4-paged-b.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r5-dense.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r5-paged-a.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r5-paged-b.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r1-dense.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r1-paged-a.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r1-paged-b.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r2-dense.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r2-paged-a.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r2-paged-b.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r3-dense.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r3-paged-a.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r3-paged-b.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r4-dense.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r4-paged-a.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r4-paged-b.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r5-dense.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r5-paged-a.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r5-paged-b.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r1-dense.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r1-paged-a.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r1-paged-b.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r2-dense.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r2-paged-a.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r2-paged-b.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r3-dense.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r3-paged-a.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r3-paged-b.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r4-dense.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r4-paged-a.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r4-paged-b.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r5-dense.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r5-paged-a.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r5-paged-b.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r1-dense.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r1-paged-a.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r1-paged-b.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r2-dense.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r2-paged-a.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r2-paged-b.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r3-dense.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r3-paged-a.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r3-paged-b.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r4-dense.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r4-paged-a.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r4-paged-b.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r5-dense.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r5-paged-a.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r5-paged-b.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/complete create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/environment.txt create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/hostgate.jsonl create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/run.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/schedule.tsv create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r1-dense.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r1-paged-a.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r1-paged-b.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r2-dense.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r2-paged-a.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r2-paged-b.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r3-dense.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r3-paged-a.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r3-paged-b.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r4-dense.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r4-paged-a.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r4-paged-b.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r5-dense.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r5-paged-a.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r5-paged-b.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r1-dense.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r1-paged-a.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r1-paged-b.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r2-dense.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r2-paged-a.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r2-paged-b.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r3-dense.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r3-paged-a.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r3-paged-b.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r4-dense.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r4-paged-a.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r4-paged-b.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r5-dense.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r5-paged-a.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r5-paged-b.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r1-dense.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r1-paged-a.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r1-paged-b.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r2-dense.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r2-paged-a.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r2-paged-b.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r3-dense.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r3-paged-a.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r3-paged-b.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r4-dense.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r4-paged-a.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r4-paged-b.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r5-dense.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r5-paged-a.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r5-paged-b.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r1-dense.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r1-paged-a.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r1-paged-b.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r2-dense.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r2-paged-a.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r2-paged-b.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r3-dense.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r3-paged-a.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r3-paged-b.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r4-dense.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r4-paged-a.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r4-paged-b.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r5-dense.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r5-paged-a.log create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r5-paged-b.log create mode 100644 docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/README.md create mode 100644 docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/SHA256SUMS create mode 100644 docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/comparison.json create mode 100644 docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/graph.stderr create mode 100644 docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/graph.stdout create mode 100644 docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/kernel.stderr create mode 100644 docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/kernel.stdout create mode 100644 docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/provenance.txt create mode 100755 docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/run.sh create mode 100644 docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/stage-deviations.log create mode 100644 docs/benchmark_results/model-owned-storage-gb10-2026-10-11.md diff --git a/docs/CONTINUOUS_BATCHING.md b/docs/CONTINUOUS_BATCHING.md index 2b487f83b..3988af1f2 100644 --- a/docs/CONTINUOUS_BATCHING.md +++ b/docs/CONTINUOUS_BATCHING.md @@ -457,6 +457,14 @@ sequence was opened. (`scheduler_real_batch_parity_tests`, gemma-3-1b-it-4bit under `MLXCEL_SDPA_DETERMINISTIC=1`) gives two concurrent greedy requests the same token streams as each request alone, on both backends. +- The [2026-10-11 GB10 paired/null storage sweep](benchmark_results/model-owned-storage-gb10-2026-10-11.md) + kept `supports_paged_decode_backend()` true for both model-owned families. + None of the six prompt/concurrency cells per family cleared its rotating + paged-repeat null spread in either direction. Therefore `auto` on a + multi-slot server and an explicit `paged` request still select paged + storage for Gemma 3 and Llama 4, including the pool allocation and + per-step mirror, while their attention continues through the dense rows + described above. - Batched decode over Turbo caches takes the dequant-first variants that single-sequence decode already used, instead of a full dequant per step. The attention is the same exact math in a different op order, so it is not diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/README.md b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/README.md new file mode 100644 index 000000000..4827e965f --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/README.md @@ -0,0 +1,45 @@ +# Raw data for model-owned-storage-gb10-2026-10-11.md + +This directory contains the complete accepted run for the Gemma 3 and Llama 4 model-owned decode-storage decision in issue #2230. The run started at `2026-10-11T04:57:35Z`, completed at `2026-10-11T07:12:28Z`, and produced 60 fresh-server arms and 180 successful client cells. + +## Contents + +- `run.sh`: fail-fast driver for the exact two-model, two-prompt, five-round, three-arm protocol. It must be invoked through the outer `gpu-lock` wrapper shown below. +- `gate.py`: per-arm fail-closed quiet-host gate. It checks the port before and after the wait, retries a post-wait race, and accepts only snapshots without active CI, build, foreign-model, memory-pressure, or NVIDIA-driver failures. +- `analyze.py`: protocol and provenance validator plus paired/null decision analyzer. +- `test_protocol.py`: lightweight synthetic fixtures for the valid protocol and rejected missing, dirty, reordered, nonmonotonic, incomplete, or backend-substituted evidence. +- `environment.txt`: contemporaneous start/end times, clean benchmark-checkout commit, release artifact identity, configured and built MLX commits, host/GPU identity, host-gate identity, and the measured gate/analyzer/driver hashes. Its `commit` key identifies the checkout used to run the sweep, not an embedded binary-source revision. +- `binary-source-provenance.json`: post-run derived verification that identifies the release binary's production source commit, proves it is an ancestor of the benchmark checkout, and records the exact four protocol-only paths in the intervening diff. It also records the preserved release-log reference and hash. This companion does not rewrite or claim to be part of the contemporaneous environment record. +- `schedule.tsv`: the exact ordered 60-arm schedule. The header plus 60 arms gives 61 lines. +- `hostgate.jsonl`: one accepted preflight snapshot followed by one accepted snapshot for each arm, in schedule order (61 records total). +- `server-*.log`: 60 nonempty server logs. Each begins with the requested dense or paged backend marker used by the analyzer. +- `client-*.txt`: 60 client logs, each with successful concurrency 1, 4, and 8 rows (180 measurement cells total). +- `run.log`: the full driver transcript, including every gate, server start, completion, and final `ALLDONE` marker. +- `analysis.txt`: analyzer output after validation, with all 12 per-model/prompt/concurrency decisions. +- `complete`: completion marker written only after all 60 arms finished successfully. +- `SHA256SUMS`: checksums for every other committed file in this directory. + +The measured release binary was built from production commit `2cb92404b153897440c4906a5548dcfb4889a9ad`. The clean checkout recorded in `environment.txt` was `35da2dbe32c731ff6645b2b32a2117af2965f332`. The earlier commit is an ancestor, and the complete intervening diff contains only `analyze.py`, `gate.py`, `run.sh`, and `test_protocol.py` in this directory. Thus the production Rust/C++/model source used by the binary is unchanged at the measurement checkout. The binary does not embed a commit; this association is explicitly derived after the run and is documented with its limitation in `binary-source-provenance.json`. + +The large Llama 4 `p512`/concurrency-1 range is traceable to the first round's first `paged-a` arm. Its client log and all surrounding arms remain unchanged in this directory. The dated report discusses the observation and its limitation without assigning an unmeasured cause. + +## Replay and validation + +`run.sh` records absolute build, model, and worktree paths because they are part of the original provenance. Replaying it requires those paths, the recorded release build dependencies, both model directories, and an idle GB10 host. The script refuses to overwrite this completed run. Use a new dated data directory for any new measurement. + +The original outer command was: + +```bash +gpu-lock run --tag bench-2230-storage -- bash \ + /home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/run.sh +``` + +Validate and reproduce the committed summary without a GPU: + +```bash +python3 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/analyze.py \ + --data-dir docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11 +python3 -W error docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/test_protocol.py +(cd docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11 && \ + sha256sum --check SHA256SUMS) +``` diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/SHA256SUMS b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/SHA256SUMS new file mode 100644 index 000000000..af20a4d86 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/SHA256SUMS @@ -0,0 +1,132 @@ +53752a55caa2f4fa517081c1695e8ca56c7fe6a31e8d898e80fe1f9ffb3f437c README.md +6e27465a51285608dbbd38798864556ee8b8cca4bf2c422fb965e45dfd60bce7 analysis.txt +f17233d3e5b5339424c050404ea81a171ad43b0abd74ebb4d4ddc681f894f3f8 analyze.py +33d55685f078a9b3f902ff8144b0aad525081d7199d94bcf7dcc77f22a0f9011 binary-source-provenance.json +21375103bda99d7465d8a29ee3929b2adb7e791abaa97322ab299f636eb9555f client-gemma3-p4096-r1-dense.txt +09f327b7653f31d7358e96004fdb801a15ff3ca3253194c1621e2f7f0d6dd1a4 client-gemma3-p4096-r1-paged-a.txt +602ef1cf6b0eab0d09fba8dec5bd40150ecf4e7475995501225710a077a844a5 client-gemma3-p4096-r1-paged-b.txt +2d259ae17c52b3b6172369d4b8888cfaa04ea51336528dc11c2a959db8423956 client-gemma3-p4096-r2-dense.txt +fb130d7d58db0d81a60574598c1b5fdfb289780f196469ced0cdf6b9f84a11a3 client-gemma3-p4096-r2-paged-a.txt +f887326227f7050687164e066f3e7a9203dcaaddf039d7b755845aa0a17ca6a8 client-gemma3-p4096-r2-paged-b.txt +ac6f468f97c8d7fbc5aa760159809edc07512898d02e2253bcc9ef6a9e7a37a7 client-gemma3-p4096-r3-dense.txt +e385ead6f2366c44759a1a205baf5ab2da30388ff7e71fb86d32103329209ca6 client-gemma3-p4096-r3-paged-a.txt +5a8dd7d0b570c19be6dccddaa64d098b54043c2857932c8b011f80ab32e7b4ea client-gemma3-p4096-r3-paged-b.txt +a1b10f06d47f174c94b390718ea392cccada4ae66305c3a8ed0ce19bf11b342e client-gemma3-p4096-r4-dense.txt +9d8b932ff7920d0d793f3ce763fb5c73ca08349bb805dcd224f41fc78d00fbec client-gemma3-p4096-r4-paged-a.txt +1f841984e590b21ed145bce3c36cb83809fbc9c311c53b17f81e2868e30a9036 client-gemma3-p4096-r4-paged-b.txt +cc374c183d72d957c9035dd5511d9f35502c9c8b20214a6bae9a6a6ea7abc113 client-gemma3-p4096-r5-dense.txt +da3c5464a40e167ead8e11e06e8c7fd3f355a26d6287d583effb4d6b346b2611 client-gemma3-p4096-r5-paged-a.txt +3ac7ae38c83104cad0ae5a5f59962587dcbe40d8d1a2aa6e5a096d1a57fe6a04 client-gemma3-p4096-r5-paged-b.txt +315daf4d873a67f3380a324d396573c2cfc02e87ae64c9bbb844475c86de809f client-gemma3-p512-r1-dense.txt +5b0dbcc4daa47802d526e1bb692080dbad4d3dd31bccc463e9d7629533bfc859 client-gemma3-p512-r1-paged-a.txt +59dc4303f35d708915587c538330d35bd30f8da6a6f9bb5d4b6f6219acefa04f client-gemma3-p512-r1-paged-b.txt +808821201119de5f97ef47572d81a0e42bd4333fa2365e6ac7df529c16cba8c9 client-gemma3-p512-r2-dense.txt +dd462f1186a5f142edd929b8bbdfa7cc331461e400dddd13a604d512013fa874 client-gemma3-p512-r2-paged-a.txt +f6df06426d5aa1565943df042bd67318e3439ab9d9513ae26f93ec322d2c4eb3 client-gemma3-p512-r2-paged-b.txt +305b08e0110ef85f4da05eb6c73912ad5deafe965a5340bbd01c38ee6225d411 client-gemma3-p512-r3-dense.txt +cb67b821c485a0c3d7256b720b5649c720302a191d13002b2ed329dfad76a963 client-gemma3-p512-r3-paged-a.txt +8375663430158203b9c98ab8f4c21de6dbf06f13f6222f530bb4972d47c347ec client-gemma3-p512-r3-paged-b.txt +76d1d1f3035149769f5532a661ce356561bcf7873cbec8461b00b07985c58bb3 client-gemma3-p512-r4-dense.txt +897a59ad2b11b4c87b27f7dde4278d8cc9062192419687d5418de623d98b3ffc client-gemma3-p512-r4-paged-a.txt +a2f7609ad2e0c1df5e0349f9304aa98a1b4b75b5c51a9389579d4c573a723042 client-gemma3-p512-r4-paged-b.txt +6be2cc9be9a6822439b8960d32555a59bca7a82e9c3f107a0843575290c0c967 client-gemma3-p512-r5-dense.txt +e1d8cb83ef1d9cec090f7d1f7c978f81aac0de3062085b10207f5d03b2138b8e client-gemma3-p512-r5-paged-a.txt +63e6a6ecaa2b15c8f01b21b7a6d03506ad23fbeb035672a69edd8e67ceb0216b client-gemma3-p512-r5-paged-b.txt +33c12bcdef7def7efcfa820c87955de74ed34bb7073e9b89e03cb6cf115efbb7 client-llama4-p4096-r1-dense.txt +08465801e328d2a15337e94739643b9f8f1d31c1847e24e56307633079a7cec5 client-llama4-p4096-r1-paged-a.txt +04931c2020134e1f0609c65211c4dec1df8e9c1ae48fc2ce1bbf8ade576321a7 client-llama4-p4096-r1-paged-b.txt +9d4b0467d70ca550c494c68b2daf15fc457e0f3d16017887f1c6df1aaf17c068 client-llama4-p4096-r2-dense.txt +282e5b1e2c74a1f9dc1e0e87b692f780433e2208fb141df7a5549ba7615f4fce client-llama4-p4096-r2-paged-a.txt +22c71b9604f52cf5895c336a16a0a874ac64af2a8ad355fa6e23c6de888dc126 client-llama4-p4096-r2-paged-b.txt +c1436e2b12c8c6fcbc646032ddaa25e81fedaa834f96496d5e4d5910c40e0efd client-llama4-p4096-r3-dense.txt +31fde7754b3d23fab81965ad0ce852b0f4f27379fbf7a669151fa5235f891c3e client-llama4-p4096-r3-paged-a.txt +960f8a875957a6845e28bca3b93f34a914f0aae448ea7fb5bbc2694b365b2b9f client-llama4-p4096-r3-paged-b.txt +6542e31e906a7283c463dd7b40c7ed4050cbee426a61d97eedcabd0ebf668c3d client-llama4-p4096-r4-dense.txt +7b31d10015ee2f9c0bca4332be54a83f01be2177e2867436de6a7526085bef5f client-llama4-p4096-r4-paged-a.txt +82dd9d9c08bc9958583a082e1bbe45d555a38c497fc192db1dfd91a6366941a0 client-llama4-p4096-r4-paged-b.txt +151bbc12e93377ac2f67e0a91836c633a3be5723549644cd4feee209d06c86b0 client-llama4-p4096-r5-dense.txt +cd1d30d43b9706cf3b626349da1c9db006da919c5cb8d8e61e6242cbf87e1ccd client-llama4-p4096-r5-paged-a.txt +c7cd013b71f11bd8fdc24bba108a4fdf3289615878acef6e7903c03d8dbb6d8c client-llama4-p4096-r5-paged-b.txt +e772dd60a43062d36f37ebdb9ed90fe3ac7ee0cad050ba38cc7ea85284ef2b8d client-llama4-p512-r1-dense.txt +7507cf3c55baa6d3186e5cbb06e1f1cfdcf92b41129ba6de1bec6105742642ec client-llama4-p512-r1-paged-a.txt +c9723faac1b0c45ceb6f5cac7ae654cc3bfcb4727e0ddad3b9c52389c6cf427a client-llama4-p512-r1-paged-b.txt +d7faa5a0b492776579e88789be69204e562e5a9ccc4c079da152ceb7dd891d82 client-llama4-p512-r2-dense.txt +abe102a422a86b7d1b38c183bcb4332f4772154b7c22970e20c576ae030cdab8 client-llama4-p512-r2-paged-a.txt +5877099da5810ce11da2a184a67f0d913333f44371204ddb050a1866afaa11d7 client-llama4-p512-r2-paged-b.txt +9f2edc96d31939a0e7786f2d942bb25cd72ec74f8a9ab3781cec160340cba491 client-llama4-p512-r3-dense.txt +a8bc891b4cbaae6d2e13540fad3dada621e1235a524131b46ecaad67004a1a1b client-llama4-p512-r3-paged-a.txt +f023cba8f523d4c9100c02d8dfbee1309bfea976d94a5a2cd88a430ae04aa9a6 client-llama4-p512-r3-paged-b.txt +abbe8225c6a43557c80e1a692d2c73b458b0f3a21010d48905eb475d080db891 client-llama4-p512-r4-dense.txt +c9c7c32408e5daa61d7762d084bf3ed8a5225d92548145199c2c4cae4bbbdcc7 client-llama4-p512-r4-paged-a.txt +1dbb91aea14984357f16cb61f7c38308c63f7284cefd935041f5327ff1244c3e client-llama4-p512-r4-paged-b.txt +61b2c017785890c09ba1e457ee72692732ebba283f759920ce10fc9ca211bf3b client-llama4-p512-r5-dense.txt +99a8cb654f29e651b414823275b61a35bdfeca404b693bc70b205f93a3fbf2d5 client-llama4-p512-r5-paged-a.txt +c317acacf2a033456ac085ab442577c397783096ecab2e1d9d23cb1d7149022c client-llama4-p512-r5-paged-b.txt +e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 complete +104f171b546eca3f1a8f8f2d0ab7fdba23a77d77c3cffae60ce9e36f910f2991 environment.txt +15a50c2718c3134750bf57e3195d294ea783cd7ae2f42940f6be1ad2dd63a607 gate.py +ea84f12f4315e03456837f262234b06a1e9298b597378795e591623c20251912 hostgate.jsonl +d7e0d1890ffc70338e714fd4f8ad07497ee7b8ce9edad0cf86012736b5db187a run.log +3b5816d50ae7cb4e5d5a5425cd31cadbcb05f33b57111c91f8e4edf35f994854 run.sh +ec628e423b2742f5fb0e0f9c26e53413b23ce215796e20f3ae38e1631257df3e schedule.tsv +99f375096980ce3769ff1c529dc64fc8e500ffb3aeaf1ad472f04aa095a01f4a server-gemma3-p4096-r1-dense.log +4ca4809af9bc7382338e285eccf2d4eba9e8b884d1a0cd5788875fd1b7168da7 server-gemma3-p4096-r1-paged-a.log +054fc6903aac719c48f3fe75e3e89d957c977bb26747a350f3eef1afaffa6051 server-gemma3-p4096-r1-paged-b.log +5d3f781a92923109c674c219a73b817ce2505128fba06179bdccbbbf1faa2b6d server-gemma3-p4096-r2-dense.log +180b5de6ed37097399e9dc08100b99a0e8e51b32d80c1e50bf7cf9da924f738d server-gemma3-p4096-r2-paged-a.log +2678311c3ad92c4278dfb17ffe8c4696471fbe5b1d4cde7931bc08ede63a91a0 server-gemma3-p4096-r2-paged-b.log +b88fa70387e15d421ee128ca6e8c96fea9609e8ef6c0d86bec386d8e5987cbf9 server-gemma3-p4096-r3-dense.log +e7c021f03b0c3477234f3065e09b68479de4cf8c2ceeeef2c2ba5e4a10a726d1 server-gemma3-p4096-r3-paged-a.log +f86242113a2a225bb6ba722e85ffc03240e90c3acf8db32a988e9dc8efe85bec server-gemma3-p4096-r3-paged-b.log +3adda534d00bcb6c5dabfddd2519ba3f42c192d385abc98287b0b45a6e470892 server-gemma3-p4096-r4-dense.log +96bcf69b4599f4743cdb36d354505285ecece6459c21b5a7a0be17a79b7e29e1 server-gemma3-p4096-r4-paged-a.log +01b23f613e5480422fc9b65e4c1fbfa9014db43149cfd15ccc0b6600f4165567 server-gemma3-p4096-r4-paged-b.log +623bd62ad1707e75e9d6f724f196141092f212006da1851bb2192b7725646746 server-gemma3-p4096-r5-dense.log +bc8f5f03ffe25c5761b5768153793fdf22cf09cdbefa9362497a2fa377fb9297 server-gemma3-p4096-r5-paged-a.log +28add5ec7465f553574525faac28c84a986c772c6ae083ba4f9a4af9054e6574 server-gemma3-p4096-r5-paged-b.log +ba4f92ab4bc09919c0f73559ea3faa13fd36dde26914c95ef1c30ed2f8c86eae server-gemma3-p512-r1-dense.log +1536fe0cca0e89182ddb280a6ab0fb6dcb85c0a95f7f4efbced9f19b8cfad92f server-gemma3-p512-r1-paged-a.log +123b5641b07aba4c569892a23cecece7367fd70d0991424669096d722df0c62f server-gemma3-p512-r1-paged-b.log +e9a15967c52ce4a1ff76e346dc92f3df8836ff7896f40be09ffc11c51e122455 server-gemma3-p512-r2-dense.log +59537cd1f653d1bde7bb56188ec61008de5d4d0dd0b3199d378cf7db283e25ac server-gemma3-p512-r2-paged-a.log +9405c3253acacffad8faf4b682de62a3f970ea9412d89b175790c599e170d13a server-gemma3-p512-r2-paged-b.log +610135b1e26e77115f7062d61281c9ba61f2f55df901e444f70415df9153c169 server-gemma3-p512-r3-dense.log +f41cb3668b468296016a5b234f232c653f87bc0a034bdc7fee88d5c0de535bc4 server-gemma3-p512-r3-paged-a.log +16c422e277535c3ca03230bc1f5499c83fd3e47072950cb4550302f35f4b10c4 server-gemma3-p512-r3-paged-b.log +50a6b6e01a9f2b344f503e8f65537e731ebfb6919743d0025f66567b97ed5a93 server-gemma3-p512-r4-dense.log +2d27e8d0fb79925d78629c9fc3d667b9fc09d4f0a1879063a67dcdcb21daae06 server-gemma3-p512-r4-paged-a.log +049a000e4c5841737f2886a6eb0028bb7106f44e1aff6acc3bccaa63ebd12ae0 server-gemma3-p512-r4-paged-b.log +75c7771bd772be38d1821509f4454f422d2d58c18b7abf9c774436d33d2d1e10 server-gemma3-p512-r5-dense.log +3364d498f9736bad65551b00c1eab355464bd20933f3c4b406ffc1b20dc54961 server-gemma3-p512-r5-paged-a.log +9348cfe5397292b25ab3e509fc02eaa254d8f2a56d6fb13e1eca448d181374dc server-gemma3-p512-r5-paged-b.log +9e5b7c523d44dd2e0bd15bbff38eb205a598d9109888253882248870e2aaee3b server-llama4-p4096-r1-dense.log +2af6593b731e3508dc9627d476921e6d04f28f56117ebb8c3acfbd6219fb87fa server-llama4-p4096-r1-paged-a.log +1b8106b6c552d3d911f81d6817b3c41402110355db2ecc3c4ad12da6e3f9c67f server-llama4-p4096-r1-paged-b.log +1974fde77f631f25c656180d4c50cca669d46e14cbb0ba4eeecaf010c115f3ef server-llama4-p4096-r2-dense.log +2cddf8181a8571018aad81f2ac765302d87c3c109c7c826d3c17684ab9d8fa5a server-llama4-p4096-r2-paged-a.log +4ad9c38651843fe0d593a6e5a21299c022397d02727d08bbec0fc519275ccdb8 server-llama4-p4096-r2-paged-b.log +d0fad1da323f52fb9a596057864f06403bb5b5f8af4a530ccb630c746b7251e7 server-llama4-p4096-r3-dense.log +63f40d12dca7bf421129506fb704261e15a227fabe4edebaab3cbf7dcc31784c server-llama4-p4096-r3-paged-a.log +781fcea815be392ecd822d1097584189884a6202e9450a80dd38149cacc51fa1 server-llama4-p4096-r3-paged-b.log +0c32aa5ac5ddaa67633fb80a8b4bf56967fd13fbb19c4a5430ae82e911939af8 server-llama4-p4096-r4-dense.log +9516d618a3db81ed9d3a9515a874afacf5292308a08be7fef30919c1ac4316d1 server-llama4-p4096-r4-paged-a.log +46c2c05238736a40b0f4227c7179baa2783bbfa8f9d15c4793e7232e32546ed0 server-llama4-p4096-r4-paged-b.log +d281ff0ae1853be6d7bb8ca0afbe7a3cd214bb477aa208e27bee2954abc74ab8 server-llama4-p4096-r5-dense.log +317a7ea390453278ac92febec897442e8c2fa4904a117bce0975abee9a23cb53 server-llama4-p4096-r5-paged-a.log +b739ccf9f05a656f060e7c5ea2d55c4b977886c3d47f0007ed07d1d37de9a25b server-llama4-p4096-r5-paged-b.log +b47ad4d644abc5f743983811279ce031d500dd2c105d2437789c2053c86e23bd server-llama4-p512-r1-dense.log +1187cfdc548d3f95c04d9ccc9e546d10ac9523a542bdf41eae39d256b5415cb8 server-llama4-p512-r1-paged-a.log +1f9d2ce943a004cccffd4683fa5027c722eaf9f643505585a5379e28b727f52d server-llama4-p512-r1-paged-b.log +77836a08fb523c91da1d168c0c581840ad62f38c4ecc3815383369269d512561 server-llama4-p512-r2-dense.log +7e5fb7971bf147456a87da4c00fe282482c933d901bb34c8fe4740a5dfe0c45f server-llama4-p512-r2-paged-a.log +87346a2d78f90ed7c5a526610abadcd9aa60f97dce1d2055f0989412be666d79 server-llama4-p512-r2-paged-b.log +2167f92e1ed1ddac693a4005de996d2d1667500370026dba59bbfdce8c110265 server-llama4-p512-r3-dense.log +f55ae55f1067cf526a4475d8ce92430e5e9a44ae9b79d5fbf8b8bec1a756483e server-llama4-p512-r3-paged-a.log +bbb7d14a80352485bcf011c49dd42dcd9a7fd25d75b7735a9f4000a0af7adf94 server-llama4-p512-r3-paged-b.log +251b2cabe3f1d6485267c2247386bf1a525c718fa05ad22a47214edb5d9fb89b server-llama4-p512-r4-dense.log +0491ea1b42eec9fc7fb18da18e99d218831b63befce3ec1f246150fb73b61ae2 server-llama4-p512-r4-paged-a.log +91e95c6dc4eef5be889acca01c6923715dde4a336d712405572df7630227795d server-llama4-p512-r4-paged-b.log +9f1defee4a9a77d98ee23e37a7c8314a1adbc99cf65cda070182560d770a0118 server-llama4-p512-r5-dense.log +473eef7371362b3038bd11fd6d58ff7bfbdcfdf56b8860d492e8fe7370898424 server-llama4-p512-r5-paged-a.log +186162f6ec56e7cd75392ff5a529ba6793684faaaaf1c5a388bebf831934feb5 server-llama4-p512-r5-paged-b.log +4c15c795123f6de85ffa424289c6f569608ac9875e18dc5dcd1429f3e51a2c8c test_protocol.py diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/analysis.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/analysis.txt new file mode 100644 index 000000000..d793195bf --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/analysis.txt @@ -0,0 +1,17 @@ +| Model | Prompt | C | paged-a median | dense median | paged-b median | dense paired delta | null spread | Cell | +|---|---:|---:|---:|---:|---:|---:|---:|---| +| gemma3 | 512 | 1 | 154.4 | 154.0 | 154.0 | +0.52% (-2.76..+3.63) | -2.37..+3.84% | unresolved | +| gemma3 | 512 | 4 | 361.0 | 363.6 | 360.6 | +1.29% (-2.11..+2.49) | -1.35..+3.52% | unresolved | +| gemma3 | 512 | 8 | 194.9 | 197.2 | 194.4 | +1.16% (+0.25..+6.37) | -0.88..+3.80% | unresolved | +| gemma3 | 4096 | 1 | 114.6 | 114.4 | 113.0 | -0.18% (-1.82..+0.26) | -2.33..+1.31% | unresolved | +| gemma3 | 4096 | 4 | 219.4 | 216.6 | 218.6 | -0.60% (-2.74..+0.54) | -3.45..+0.50% | unresolved | +| gemma3 | 4096 | 8 | 184.4 | 185.8 | 184.0 | +0.49% (-0.53..+1.98) | -3.13..+1.91% | unresolved | +| llama4 | 512 | 1 | 11.7 | 11.8 | 11.8 | +0.85% (+0.00..+83.08) | -0.85..+81.54% | unresolved | +| llama4 | 512 | 4 | 31.0 | 30.8 | 30.7 | +0.65% (-3.53..+2.61) | -0.97..+3.53% | unresolved | +| llama4 | 512 | 8 | 17.8 | 17.9 | 17.7 | +0.00% (-1.69..+13.29) | -1.13..+10.76% | unresolved | +| llama4 | 4096 | 1 | 8.0 | 8.0 | 8.0 | +1.25% (+0.00..+1.27) | +0.00..+2.53% | unresolved | +| llama4 | 4096 | 4 | 22.9 | 23.1 | 23.0 | +0.44% (-0.43..+2.25) | -2.99..+4.05% | unresolved | +| llama4 | 4096 | 8 | 14.7 | 14.7 | 14.5 | +0.68% (-0.68..+3.55) | -2.03..+2.84% | unresolved | + +gemma3: supports_paged_decode_backend=true (dense_clear_cell=False, paged_clear_cell=False) +llama4: supports_paged_decode_backend=true (dense_clear_cell=False, paged_clear_cell=False) diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/binary-source-provenance.json b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/binary-source-provenance.json new file mode 100644 index 000000000..5c3ace7c1 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/binary-source-provenance.json @@ -0,0 +1,26 @@ +{ + "schema_version": 1, + "evidence_kind": "post-run-derived-verification", + "verified_utc": "2026-10-11T07:20:08Z", + "checkout_commit": "35da2dbe32c731ff6645b2b32a2117af2965f332", + "environment_commit_semantics": "environment.txt commit is the clean benchmark checkout at run start", + "binary_source_commit": "2cb92404b153897440c4906a5548dcfb4889a9ad", + "binary_sha256": "0cc49217021dab12b5add25dc2bb8de7e90c405c0b406607b244a8060d3ed800", + "binary_size_bytes": 108106312, + "binary_mtime_utc": "2026-10-11T03:25:49Z", + "release_build_log": "/tmp/mlxcel-auto-plan/2230-release-build.log", + "release_build_log_sha256": "037e0fb240ec19853a63eecd66a6d4d7045c510e4bd0f3e13ab558b48b13e31c", + "release_build_result": "Finished release profile in 19m 43s", + "ancestor_command": "git merge-base --is-ancestor 2cb92404b153897440c4906a5548dcfb4889a9ad 35da2dbe32c731ff6645b2b32a2117af2965f332", + "ancestor_command_exit_code": 0, + "diff_command": "git diff --name-only --no-renames 2cb92404b153897440c4906a5548dcfb4889a9ad..35da2dbe32c731ff6645b2b32a2117af2965f332", + "intervening_paths": [ + "docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/analyze.py", + "docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/gate.py", + "docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/run.sh", + "docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/test_protocol.py" + ], + "production_source_equivalent": true, + "derivation": "The preserved release-build session identifies 2cb92404 as the binary build source. That commit is an ancestor of the clean measurement checkout, and the complete intervening diff contains only the four listed benchmark-protocol files. No Rust, C++, model, or build-definition path changed between the binary source and measurement checkout.", + "limitation": "The release binary does not embed a source commit. The binary-source association is reconstructed after the run from the preserved build session, artifact hash and mtime, commit ancestry, and exact intervening path set; it is not a contemporaneous field in environment.txt. The original environment and measured scripts remain byte-for-byte unchanged." +} diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r1-dense.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r1-dense.txt new file mode 100644 index 000000000..a539ea31d --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r1-dense.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+0.576 + batch steps: occupancy=1.00 decode_step_ms=4.54 (predicted_seconds/steps=4.54 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+4 tokens_predicted_seconds=+7.005 + batch steps: occupancy=4.00 decode_step_ms=13.79 (predicted_seconds/steps=55.16 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+43.839 + batch steps: occupancy=8.00 decode_step_ms=43.15 (predicted_seconds/steps=345.19 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 547.1 547.1 217.2 113.1 + 4 4 0 642.9 1016.9 74.2 213.0 + 8 8 0 53.9 85.6 23.1 184.7 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r1-paged-a.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r1-paged-a.txt new file mode 100644 index 000000000..96a70fb66 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r1-paged-a.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+0.564 + batch steps: occupancy=1.00 decode_step_ms=4.44 (predicted_seconds/steps=4.44 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+4 tokens_predicted_seconds=+6.718 + batch steps: occupancy=4.00 decode_step_ms=13.22 (predicted_seconds/steps=52.90 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+43.900 + batch steps: occupancy=8.00 decode_step_ms=43.21 (predicted_seconds/steps=345.67 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 542.0 542.0 223.3 115.2 + 4 4 0 650.8 1020.9 77.5 219.0 + 8 8 0 54.4 84.7 23.1 184.4 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r1-paged-b.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r1-paged-b.txt new file mode 100644 index 000000000..ebcfa48bf --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r1-paged-b.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+0.574 + batch steps: occupancy=1.00 decode_step_ms=4.52 (predicted_seconds/steps=4.52 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+4 tokens_predicted_seconds=+6.754 + batch steps: occupancy=4.00 decode_step_ms=13.30 (predicted_seconds/steps=53.18 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+44.051 + batch steps: occupancy=8.00 decode_step_ms=43.36 (predicted_seconds/steps=346.86 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 550.6 550.6 217.8 112.9 + 4 4 0 644.3 1016.4 76.9 218.6 + 8 8 0 55.8 89.7 23.0 183.8 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r2-dense.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r2-dense.txt new file mode 100644 index 000000000..77699b816 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r2-dense.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+0.559 + batch steps: occupancy=1.00 decode_step_ms=4.40 (predicted_seconds/steps=4.40 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+4 tokens_predicted_seconds=+6.642 + batch steps: occupancy=4.00 decode_step_ms=13.07 (predicted_seconds/steps=52.30 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+43.325 + batch steps: occupancy=8.00 decode_step_ms=42.64 (predicted_seconds/steps=341.14 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 545.0 545.0 223.4 115.0 + 4 4 0 642.8 1014.7 78.4 221.5 + 8 8 0 53.4 84.8 23.4 186.9 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r2-paged-a.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r2-paged-a.txt new file mode 100644 index 000000000..0b3375d2d --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r2-paged-a.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+0.560 + batch steps: occupancy=1.00 decode_step_ms=4.41 (predicted_seconds/steps=4.41 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+4 tokens_predicted_seconds=+6.701 + batch steps: occupancy=4.00 decode_step_ms=13.19 (predicted_seconds/steps=52.76 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+43.097 + batch steps: occupancy=8.00 decode_step_ms=42.42 (predicted_seconds/steps=339.35 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 541.9 541.9 224.9 115.7 + 4 4 0 640.3 1009.0 77.7 220.3 + 8 8 0 53.1 84.3 23.5 187.9 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r2-paged-b.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r2-paged-b.txt new file mode 100644 index 000000000..322fd3131 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r2-paged-b.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+0.578 + batch steps: occupancy=1.00 decode_step_ms=4.55 (predicted_seconds/steps=4.55 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+4 tokens_predicted_seconds=+7.002 + batch steps: occupancy=4.00 decode_step_ms=13.78 (predicted_seconds/steps=55.13 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+44.028 + batch steps: occupancy=8.00 decode_step_ms=43.33 (predicted_seconds/steps=346.68 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 549.4 549.4 217.8 113.0 + 4 4 0 648.4 1021.3 74.2 212.7 + 8 8 0 53.1 85.3 23.1 184.0 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r3-dense.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r3-dense.txt new file mode 100644 index 000000000..1a41f669e --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r3-dense.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+0.561 + batch steps: occupancy=1.00 decode_step_ms=4.42 (predicted_seconds/steps=4.42 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+4 tokens_predicted_seconds=+6.852 + batch steps: occupancy=4.00 decode_step_ms=13.49 (predicted_seconds/steps=53.95 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+42.892 + batch steps: occupancy=8.00 decode_step_ms=42.22 (predicted_seconds/steps=337.73 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 557.6 557.6 223.3 113.6 + 4 4 0 642.6 1013.5 75.9 216.6 + 8 8 0 54.9 88.3 23.7 188.7 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r3-paged-a.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r3-paged-a.txt new file mode 100644 index 000000000..98cad85c4 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r3-paged-a.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+0.560 + batch steps: occupancy=1.00 decode_step_ms=4.41 (predicted_seconds/steps=4.41 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+4 tokens_predicted_seconds=+6.703 + batch steps: occupancy=4.00 decode_step_ms=13.19 (predicted_seconds/steps=52.78 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+43.388 + batch steps: occupancy=8.00 decode_step_ms=42.70 (predicted_seconds/steps=341.64 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 556.2 556.2 223.5 113.8 + 4 4 0 645.0 1018.7 77.6 219.7 + 8 8 0 54.7 87.7 23.4 186.6 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r3-paged-b.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r3-paged-b.txt new file mode 100644 index 000000000..db535d612 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r3-paged-b.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+0.564 + batch steps: occupancy=1.00 decode_step_ms=4.44 (predicted_seconds/steps=4.44 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+4 tokens_predicted_seconds=+6.693 + batch steps: occupancy=4.00 decode_step_ms=13.18 (predicted_seconds/steps=52.70 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+43.334 + batch steps: occupancy=8.00 decode_step_ms=42.65 (predicted_seconds/steps=341.21 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 544.8 544.8 222.9 114.9 + 4 4 0 647.8 1021.1 77.8 219.7 + 8 8 0 54.1 85.7 23.4 186.8 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r4-dense.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r4-dense.txt new file mode 100644 index 000000000..853234844 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r4-dense.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+0.567 + batch steps: occupancy=1.00 decode_step_ms=4.46 (predicted_seconds/steps=4.46 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+4 tokens_predicted_seconds=+6.835 + batch steps: occupancy=4.00 decode_step_ms=13.45 (predicted_seconds/steps=53.82 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+43.880 + batch steps: occupancy=8.00 decode_step_ms=43.19 (predicted_seconds/steps=345.51 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 547.8 547.8 222.3 114.4 + 4 4 0 645.8 1016.6 76.1 216.6 + 8 8 0 53.4 85.8 23.1 184.6 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r4-paged-a.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r4-paged-a.txt new file mode 100644 index 000000000..9200e80f9 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r4-paged-a.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+0.571 + batch steps: occupancy=1.00 decode_step_ms=4.50 (predicted_seconds/steps=4.50 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+4 tokens_predicted_seconds=+6.777 + batch steps: occupancy=4.00 decode_step_ms=13.34 (predicted_seconds/steps=53.36 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+44.075 + batch steps: occupancy=8.00 decode_step_ms=43.38 (predicted_seconds/steps=347.05 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 541.9 541.9 220.8 114.6 + 4 4 0 646.0 1020.5 76.8 217.9 + 8 8 0 55.2 87.2 23.0 183.7 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r4-paged-b.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r4-paged-b.txt new file mode 100644 index 000000000..e04846d71 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r4-paged-b.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+0.552 + batch steps: occupancy=1.00 decode_step_ms=4.35 (predicted_seconds/steps=4.35 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+4 tokens_predicted_seconds=+6.750 + batch steps: occupancy=4.00 decode_step_ms=13.29 (predicted_seconds/steps=53.15 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+43.240 + batch steps: occupancy=8.00 decode_step_ms=42.56 (predicted_seconds/steps=340.47 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 545.0 545.0 227.9 116.1 + 4 4 0 642.5 1014.9 77.1 219.0 + 8 8 0 54.9 85.5 23.5 187.2 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r5-dense.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r5-dense.txt new file mode 100644 index 000000000..ab3436551 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r5-dense.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+0.564 + batch steps: occupancy=1.00 decode_step_ms=4.44 (predicted_seconds/steps=4.44 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+4 tokens_predicted_seconds=+6.700 + batch steps: occupancy=4.00 decode_step_ms=13.19 (predicted_seconds/steps=52.76 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+43.552 + batch steps: occupancy=8.00 decode_step_ms=42.87 (predicted_seconds/steps=342.93 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 544.9 544.9 222.1 114.6 + 4 4 0 644.5 1019.8 77.7 219.9 + 8 8 0 54.6 85.3 23.3 185.8 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r5-paged-a.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r5-paged-a.txt new file mode 100644 index 000000000..f5ed980d3 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r5-paged-a.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+0.569 + batch steps: occupancy=1.00 decode_step_ms=4.48 (predicted_seconds/steps=4.48 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+4 tokens_predicted_seconds=+6.733 + batch steps: occupancy=4.00 decode_step_ms=13.25 (predicted_seconds/steps=53.02 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+44.423 + batch steps: occupancy=8.00 decode_step_ms=43.72 (predicted_seconds/steps=349.79 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 545.7 545.7 221.3 114.3 + 4 4 0 641.1 1012.2 77.2 219.4 + 8 8 0 57.3 88.5 22.8 182.2 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r5-paged-b.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r5-paged-b.txt new file mode 100644 index 000000000..c5346d33e --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p4096-r5-paged-b.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+0.576 + batch steps: occupancy=1.00 decode_step_ms=4.54 (predicted_seconds/steps=4.54 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+4 tokens_predicted_seconds=+6.968 + batch steps: occupancy=4.00 decode_step_ms=13.72 (predicted_seconds/steps=54.87 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+45.896 + batch steps: occupancy=8.00 decode_step_ms=45.17 (predicted_seconds/steps=361.39 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 549.2 549.2 217.0 112.8 + 4 4 0 643.6 1014.1 74.6 214.1 + 8 8 0 55.7 89.2 22.1 176.5 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r1-dense.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r1-dense.txt new file mode 100644 index 000000000..3bed79ce9 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r1-dense.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+0.515 + batch steps: occupancy=1.00 decode_step_ms=4.06 (predicted_seconds/steps=4.06 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+5.088 + batch steps: occupancy=4.00 decode_step_ms=10.02 (predicted_seconds/steps=40.06 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+40.410 + batch steps: occupancy=8.00 decode_step_ms=39.77 (predicted_seconds/steps=318.19 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 313.3 313.3 245.3 154.0 + 4 4 0 107.7 159.9 99.7 370.0 + 8 8 0 38.7 61.9 25.1 200.9 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r1-paged-a.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r1-paged-a.txt new file mode 100644 index 000000000..f4465d7e4 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r1-paged-a.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+0.511 + batch steps: occupancy=1.00 decode_step_ms=4.02 (predicted_seconds/steps=4.02 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+5.223 + batch steps: occupancy=4.00 decode_step_ms=10.28 (predicted_seconds/steps=41.13 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+40.905 + batch steps: occupancy=8.00 decode_step_ms=40.26 (predicted_seconds/steps=322.09 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 348.2 348.2 247.4 148.6 + 4 4 0 110.4 163.3 97.3 361.0 + 8 8 0 37.4 62.8 24.8 198.6 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r1-paged-b.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r1-paged-b.txt new file mode 100644 index 000000000..aaa3d92e0 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r1-paged-b.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+0.517 + batch steps: occupancy=1.00 decode_step_ms=4.07 (predicted_seconds/steps=4.07 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+5.039 + batch steps: occupancy=4.00 decode_step_ms=9.92 (predicted_seconds/steps=39.68 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+41.179 + batch steps: occupancy=8.00 decode_step_ms=40.53 (predicted_seconds/steps=324.24 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 311.3 311.3 245.1 154.3 + 4 4 0 107.4 157.9 100.7 373.7 + 8 8 0 37.3 60.7 24.7 197.3 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r2-dense.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r2-dense.txt new file mode 100644 index 000000000..422f153e9 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r2-dense.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+0.528 + batch steps: occupancy=1.00 decode_step_ms=4.16 (predicted_seconds/steps=4.16 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+5.187 + batch steps: occupancy=4.00 decode_step_ms=10.21 (predicted_seconds/steps=40.84 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+41.225 + batch steps: occupancy=8.00 decode_step_ms=40.58 (predicted_seconds/steps=324.61 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 315.3 315.3 240.0 151.6 + 4 4 0 108.4 160.7 97.9 363.6 + 8 8 0 36.8 60.4 24.6 197.0 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r2-paged-a.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r2-paged-a.txt new file mode 100644 index 000000000..43f3475b4 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r2-paged-a.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+0.513 + batch steps: occupancy=1.00 decode_step_ms=4.04 (predicted_seconds/steps=4.04 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+5.222 + batch steps: occupancy=4.00 decode_step_ms=10.28 (predicted_seconds/steps=41.12 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+41.351 + batch steps: occupancy=8.00 decode_step_ms=40.70 (predicted_seconds/steps=325.60 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 306.4 306.4 246.8 155.9 + 4 4 0 108.2 158.9 97.3 361.5 + 8 8 0 39.6 65.5 24.6 196.5 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r2-paged-b.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r2-paged-b.txt new file mode 100644 index 000000000..f1ec126b5 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r2-paged-b.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+0.521 + batch steps: occupancy=1.00 decode_step_ms=4.10 (predicted_seconds/steps=4.10 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+5.215 + batch steps: occupancy=4.00 decode_step_ms=10.27 (predicted_seconds/steps=41.06 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+40.867 + batch steps: occupancy=8.00 decode_step_ms=40.22 (predicted_seconds/steps=321.79 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 317.7 317.7 242.6 152.2 + 4 4 0 111.7 164.6 97.4 361.0 + 8 8 0 39.2 64.0 24.8 198.7 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r3-dense.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r3-dense.txt new file mode 100644 index 000000000..91efe0366 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r3-dense.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+0.528 + batch steps: occupancy=1.00 decode_step_ms=4.16 (predicted_seconds/steps=4.16 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+5.301 + batch steps: occupancy=4.00 decode_step_ms=10.44 (predicted_seconds/steps=41.74 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+40.894 + batch steps: occupancy=8.00 decode_step_ms=40.25 (predicted_seconds/steps=322.00 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 308.7 308.7 240.1 152.8 + 4 4 0 108.1 161.6 95.8 356.5 + 8 8 0 37.2 60.5 24.8 198.6 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r3-paged-a.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r3-paged-a.txt new file mode 100644 index 000000000..90fe13010 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r3-paged-a.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+0.519 + batch steps: occupancy=1.00 decode_step_ms=4.09 (predicted_seconds/steps=4.09 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+5.180 + batch steps: occupancy=4.00 decode_step_ms=10.20 (predicted_seconds/steps=40.79 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+43.535 + batch steps: occupancy=8.00 decode_step_ms=42.85 (predicted_seconds/steps=342.80 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 305.2 305.2 244.1 155.1 + 4 4 0 107.1 159.3 98.0 364.2 + 8 8 0 38.4 62.1 23.3 186.7 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r3-paged-b.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r3-paged-b.txt new file mode 100644 index 000000000..8413fee1c --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r3-paged-b.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+0.516 + batch steps: occupancy=1.00 decode_step_ms=4.06 (predicted_seconds/steps=4.06 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+5.253 + batch steps: occupancy=4.00 decode_step_ms=10.34 (predicted_seconds/steps=41.36 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+41.897 + batch steps: occupancy=8.00 decode_step_ms=41.24 (predicted_seconds/steps=329.90 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 312.6 312.6 245.0 154.0 + 4 4 0 108.7 160.6 96.6 359.3 + 8 8 0 39.3 64.4 24.2 193.8 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r4-dense.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r4-dense.txt new file mode 100644 index 000000000..d93189983 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r4-dense.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+0.514 + batch steps: occupancy=1.00 decode_step_ms=4.05 (predicted_seconds/steps=4.05 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+5.223 + batch steps: occupancy=4.00 decode_step_ms=10.28 (predicted_seconds/steps=41.13 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+41.179 + batch steps: occupancy=8.00 decode_step_ms=40.53 (predicted_seconds/steps=324.24 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 309.2 309.2 246.3 155.2 + 4 4 0 109.0 162.5 97.2 361.2 + 8 8 0 39.8 64.3 24.7 197.2 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r4-paged-a.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r4-paged-a.txt new file mode 100644 index 000000000..2c4129dc3 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r4-paged-a.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+0.515 + batch steps: occupancy=1.00 decode_step_ms=4.06 (predicted_seconds/steps=4.06 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+5.289 + batch steps: occupancy=4.00 decode_step_ms=10.41 (predicted_seconds/steps=41.65 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+41.670 + batch steps: occupancy=8.00 decode_step_ms=41.01 (predicted_seconds/steps=328.11 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 313.6 313.6 246.3 154.4 + 4 4 0 110.2 163.8 96.0 356.6 + 8 8 0 38.1 62.9 24.4 194.9 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r4-paged-b.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r4-paged-b.txt new file mode 100644 index 000000000..cc8b80886 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r4-paged-b.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+0.518 + batch steps: occupancy=1.00 decode_step_ms=4.08 (predicted_seconds/steps=4.08 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+5.368 + batch steps: occupancy=4.00 decode_step_ms=10.57 (predicted_seconds/steps=42.27 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+41.792 + batch steps: occupancy=8.00 decode_step_ms=41.13 (predicted_seconds/steps=329.07 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 305.5 305.5 245.0 155.3 + 4 4 0 109.0 162.1 94.6 352.2 + 8 8 0 38.3 65.7 24.3 194.4 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r5-dense.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r5-dense.txt new file mode 100644 index 000000000..8a47f5b99 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r5-dense.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+0.510 + batch steps: occupancy=1.00 decode_step_ms=4.02 (predicted_seconds/steps=4.02 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+5.179 + batch steps: occupancy=4.00 decode_step_ms=10.19 (predicted_seconds/steps=40.78 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+41.815 + batch steps: occupancy=8.00 decode_step_ms=41.16 (predicted_seconds/steps=329.25 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 308.2 308.2 248.3 156.2 + 4 4 0 109.9 163.1 98.1 363.8 + 8 8 0 38.8 63.5 24.3 194.3 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r5-paged-a.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r5-paged-a.txt new file mode 100644 index 000000000..95e9f8969 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r5-paged-a.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+0.520 + batch steps: occupancy=1.00 decode_step_ms=4.09 (predicted_seconds/steps=4.09 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+5.275 + batch steps: occupancy=4.00 decode_step_ms=10.38 (predicted_seconds/steps=41.54 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+41.928 + batch steps: occupancy=8.00 decode_step_ms=41.27 (predicted_seconds/steps=330.14 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 311.3 311.3 243.0 153.5 + 4 4 0 110.7 163.6 96.3 357.5 + 8 8 0 38.3 63.7 24.2 193.7 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r5-paged-b.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r5-paged-b.txt new file mode 100644 index 000000000..f2590512c --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-gemma3-p512-r5-paged-b.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: gemma-3-1b-it-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+0.518 + batch steps: occupancy=1.00 decode_step_ms=4.08 (predicted_seconds/steps=4.08 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+5.216 + batch steps: occupancy=4.00 decode_step_ms=10.27 (predicted_seconds/steps=41.07 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+42.302 + batch steps: occupancy=8.00 decode_step_ms=41.64 (predicted_seconds/steps=333.09 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 312.3 312.3 244.7 154.0 + 4 4 0 112.2 166.0 97.3 360.6 + 8 8 0 39.0 65.1 24.0 192.0 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r1-dense.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r1-dense.txt new file mode 100644 index 000000000..6d5718b63 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r1-dense.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+7.704 + batch steps: occupancy=1.00 decode_step_ms=60.66 (predicted_seconds/steps=60.66 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+87.283 + batch steps: occupancy=4.00 decode_step_ms=171.82 (predicted_seconds/steps=687.27 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+555.555 + batch steps: occupancy=8.00 decode_step_ms=546.81 (predicted_seconds/steps=4374.45 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 8242.4 8242.4 16.5 8.0 + 4 4 0 378.5 484.0 5.8 23.1 + 8 8 0 549.0 858.3 1.8 14.6 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r1-paged-a.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r1-paged-a.txt new file mode 100644 index 000000000..381483fe4 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r1-paged-a.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+7.850 + batch steps: occupancy=1.00 decode_step_ms=61.81 (predicted_seconds/steps=61.81 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+87.296 + batch steps: occupancy=4.00 decode_step_ms=171.84 (predicted_seconds/steps=687.37 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+575.314 + batch steps: occupancy=8.00 decode_step_ms=566.25 (predicted_seconds/steps=4530.03 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 8341.2 8341.2 16.2 7.9 + 4 4 0 379.7 487.2 5.8 23.1 + 8 8 0 567.2 904.7 1.8 14.1 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r1-paged-b.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r1-paged-b.txt new file mode 100644 index 000000000..d5a0ec095 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r1-paged-b.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+7.771 + batch steps: occupancy=1.00 decode_step_ms=61.19 (predicted_seconds/steps=61.19 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+86.767 + batch steps: occupancy=4.00 decode_step_ms=170.80 (predicted_seconds/steps=683.20 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+562.047 + batch steps: occupancy=8.00 decode_step_ms=553.20 (predicted_seconds/steps=4425.57 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 8351.2 8351.2 16.3 7.9 + 4 4 0 379.9 487.3 5.9 23.2 + 8 8 0 549.1 899.9 1.8 14.5 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r2-dense.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r2-dense.txt new file mode 100644 index 000000000..6725d27bc --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r2-dense.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+7.658 + batch steps: occupancy=1.00 decode_step_ms=60.30 (predicted_seconds/steps=60.30 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+88.832 + batch steps: occupancy=4.00 decode_step_ms=174.87 (predicted_seconds/steps=699.46 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+554.225 + batch steps: occupancy=8.00 decode_step_ms=545.50 (predicted_seconds/steps=4363.98 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 8150.2 8150.2 16.6 8.1 + 4 4 0 375.8 482.3 5.7 22.7 + 8 8 0 591.2 954.9 1.8 14.7 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r2-paged-a.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r2-paged-a.txt new file mode 100644 index 000000000..cbb170448 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r2-paged-a.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+7.774 + batch steps: occupancy=1.00 decode_step_ms=61.21 (predicted_seconds/steps=61.21 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+90.671 + batch steps: occupancy=4.00 decode_step_ms=178.49 (predicted_seconds/steps=713.94 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+549.513 + batch steps: occupancy=8.00 decode_step_ms=540.86 (predicted_seconds/steps=4326.87 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 8236.5 8236.5 16.3 8.0 + 4 4 0 390.8 500.9 5.6 22.2 + 8 8 0 571.3 917.3 1.8 14.8 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r2-paged-b.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r2-paged-b.txt new file mode 100644 index 000000000..b1c1b76fa --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r2-paged-b.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+7.771 + batch steps: occupancy=1.00 decode_step_ms=61.19 (predicted_seconds/steps=61.19 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+87.130 + batch steps: occupancy=4.00 decode_step_ms=171.52 (predicted_seconds/steps=686.06 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+559.801 + batch steps: occupancy=8.00 decode_step_ms=550.99 (predicted_seconds/steps=4407.88 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 8281.8 8281.8 16.3 8.0 + 4 4 0 386.3 495.4 5.8 23.1 + 8 8 0 558.4 911.2 1.8 14.5 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r3-dense.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r3-dense.txt new file mode 100644 index 000000000..756d28aa3 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r3-dense.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+7.792 + batch steps: occupancy=1.00 decode_step_ms=61.35 (predicted_seconds/steps=61.35 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+86.392 + batch steps: occupancy=4.00 decode_step_ms=170.06 (predicted_seconds/steps=680.25 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+550.118 + batch steps: occupancy=8.00 decode_step_ms=541.45 (predicted_seconds/steps=4331.64 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 8239.7 8239.7 16.3 8.0 + 4 4 0 380.6 487.5 5.9 23.3 + 8 8 0 555.3 919.7 1.8 14.8 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r3-paged-a.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r3-paged-a.txt new file mode 100644 index 000000000..ba6e7feb8 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r3-paged-a.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+7.802 + batch steps: occupancy=1.00 decode_step_ms=61.43 (predicted_seconds/steps=61.43 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+87.743 + batch steps: occupancy=4.00 decode_step_ms=172.72 (predicted_seconds/steps=690.89 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+558.019 + batch steps: occupancy=8.00 decode_step_ms=549.23 (predicted_seconds/steps=4393.85 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 8308.2 8308.2 16.3 7.9 + 4 4 0 390.7 501.6 5.8 22.9 + 8 8 0 560.0 888.1 1.8 14.6 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r3-paged-b.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r3-paged-b.txt new file mode 100644 index 000000000..e4c53cc20 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r3-paged-b.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+7.741 + batch steps: occupancy=1.00 decode_step_ms=60.95 (predicted_seconds/steps=60.95 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+87.338 + batch steps: occupancy=4.00 decode_step_ms=171.93 (predicted_seconds/steps=687.70 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+560.279 + batch steps: occupancy=8.00 decode_step_ms=551.46 (predicted_seconds/steps=4411.65 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 8154.1 8154.1 16.4 8.1 + 4 4 0 378.2 483.0 5.8 23.0 + 8 8 0 547.6 888.3 1.8 14.5 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r4-dense.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r4-dense.txt new file mode 100644 index 000000000..13d134449 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r4-dense.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+7.794 + batch steps: occupancy=1.00 decode_step_ms=61.37 (predicted_seconds/steps=61.37 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+86.196 + batch steps: occupancy=4.00 decode_step_ms=169.68 (predicted_seconds/steps=678.71 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+550.041 + batch steps: occupancy=8.00 decode_step_ms=541.38 (predicted_seconds/steps=4331.03 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 8145.5 8145.5 16.3 8.0 + 4 4 0 374.0 478.4 5.9 23.3 + 8 8 0 540.3 872.8 1.8 14.8 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r4-paged-a.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r4-paged-a.txt new file mode 100644 index 000000000..14511506e --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r4-paged-a.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+7.735 + batch steps: occupancy=1.00 decode_step_ms=60.91 (predicted_seconds/steps=60.91 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+86.095 + batch steps: occupancy=4.00 decode_step_ms=169.48 (predicted_seconds/steps=677.91 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+554.484 + batch steps: occupancy=8.00 decode_step_ms=545.75 (predicted_seconds/steps=4366.02 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 8295.4 8295.4 16.4 8.0 + 4 4 0 377.4 482.2 5.9 23.4 + 8 8 0 547.7 884.9 1.8 14.7 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r4-paged-b.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r4-paged-b.txt new file mode 100644 index 000000000..9b0d4a2a2 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r4-paged-b.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+7.702 + batch steps: occupancy=1.00 decode_step_ms=60.65 (predicted_seconds/steps=60.65 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+88.698 + batch steps: occupancy=4.00 decode_step_ms=174.60 (predicted_seconds/steps=698.41 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+542.200 + batch steps: occupancy=8.00 decode_step_ms=533.66 (predicted_seconds/steps=4269.29 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 8200.8 8200.8 16.5 8.0 + 4 4 0 377.6 483.2 5.7 22.7 + 8 8 0 569.6 932.7 1.9 15.0 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r5-dense.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r5-dense.txt new file mode 100644 index 000000000..f53b95d4b --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r5-dense.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+7.725 + batch steps: occupancy=1.00 decode_step_ms=60.83 (predicted_seconds/steps=60.83 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+88.900 + batch steps: occupancy=4.00 decode_step_ms=175.00 (predicted_seconds/steps=700.00 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+553.814 + batch steps: occupancy=8.00 decode_step_ms=545.09 (predicted_seconds/steps=4360.74 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 8141.1 8141.1 16.4 8.1 + 4 4 0 375.0 479.8 5.7 22.6 + 8 8 0 563.0 919.4 1.8 14.7 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r5-paged-a.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r5-paged-a.txt new file mode 100644 index 000000000..d770afbd4 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r5-paged-a.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+7.706 + batch steps: occupancy=1.00 decode_step_ms=60.68 (predicted_seconds/steps=60.68 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+89.349 + batch steps: occupancy=4.00 decode_step_ms=175.88 (predicted_seconds/steps=703.54 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+548.808 + batch steps: occupancy=8.00 decode_step_ms=540.17 (predicted_seconds/steps=4321.32 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 8248.0 8248.0 16.5 8.0 + 4 4 0 382.1 488.5 5.7 22.5 + 8 8 0 557.3 881.9 1.9 14.8 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r5-paged-b.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r5-paged-b.txt new file mode 100644 index 000000000..9c4dd7558 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p4096-r5-paged-b.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~4096 tokens (20735 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+1 tokens_predicted_seconds=+7.738 + batch steps: occupancy=1.00 decode_step_ms=60.93 (predicted_seconds/steps=60.93 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+87.461 + batch steps: occupancy=4.00 decode_step_ms=172.17 (predicted_seconds/steps=688.67 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+555.304 + batch steps: occupancy=8.00 decode_step_ms=546.56 (predicted_seconds/steps=4372.47 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 8214.9 8214.9 16.4 8.0 + 4 4 0 386.7 495.2 5.8 23.0 + 8 8 0 554.9 900.4 1.8 14.6 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r1-dense.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r1-dense.txt new file mode 100644 index 000000000..94dd774cc --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r1-dense.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+6.706 + batch steps: occupancy=1.00 decode_step_ms=52.80 (predicted_seconds/steps=52.80 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+66.631 + batch steps: occupancy=4.00 decode_step_ms=131.16 (predicted_seconds/steps=524.65 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+453.087 + batch steps: occupancy=8.00 decode_step_ms=445.95 (predicted_seconds/steps=3567.61 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 4072.6 4072.6 18.9 11.9 + 4 4 0 315.5 398.8 7.6 30.1 + 8 8 0 427.0 619.1 2.2 17.9 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r1-paged-a.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r1-paged-a.txt new file mode 100644 index 000000000..651f23ae7 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r1-paged-a.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+7.121 + batch steps: occupancy=1.00 decode_step_ms=56.07 (predicted_seconds/steps=56.07 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+64.370 + batch steps: occupancy=4.00 decode_step_ms=126.71 (predicted_seconds/steps=506.85 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+514.195 + batch steps: occupancy=8.00 decode_step_ms=506.10 (predicted_seconds/steps=4048.78 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 12690.4 12690.4 17.8 6.5 + 4 4 0 325.6 407.6 7.9 31.2 + 8 8 0 436.7 634.4 2.0 15.8 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r1-paged-b.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r1-paged-b.txt new file mode 100644 index 000000000..54d89d1cc --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r1-paged-b.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+6.723 + batch steps: occupancy=1.00 decode_step_ms=52.94 (predicted_seconds/steps=52.94 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+62.075 + batch steps: occupancy=4.00 decode_step_ms=122.19 (predicted_seconds/steps=488.78 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+463.536 + batch steps: occupancy=8.00 decode_step_ms=456.24 (predicted_seconds/steps=3649.89 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 4128.2 4128.2 18.9 11.8 + 4 4 0 325.0 410.2 8.2 32.3 + 8 8 0 429.6 625.2 2.2 17.5 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r2-dense.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r2-dense.txt new file mode 100644 index 000000000..19964e4ac --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r2-dense.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+6.741 + batch steps: occupancy=1.00 decode_step_ms=53.08 (predicted_seconds/steps=53.08 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+65.145 + batch steps: occupancy=4.00 decode_step_ms=128.24 (predicted_seconds/steps=512.95 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+449.201 + batch steps: occupancy=8.00 decode_step_ms=442.13 (predicted_seconds/steps=3537.02 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 4142.2 4142.2 18.8 11.8 + 4 4 0 315.9 396.5 7.8 30.8 + 8 8 0 432.1 626.5 2.3 18.1 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r2-paged-a.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r2-paged-a.txt new file mode 100644 index 000000000..3390c7bd5 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r2-paged-a.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+6.707 + batch steps: occupancy=1.00 decode_step_ms=52.81 (predicted_seconds/steps=52.81 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+65.870 + batch steps: occupancy=4.00 decode_step_ms=129.67 (predicted_seconds/steps=518.66 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+449.328 + batch steps: occupancy=8.00 decode_step_ms=442.25 (predicted_seconds/steps=3538.02 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 4125.5 4125.5 18.9 11.8 + 4 4 0 316.1 397.7 7.7 30.5 + 8 8 0 438.4 638.5 2.3 18.1 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r2-paged-b.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r2-paged-b.txt new file mode 100644 index 000000000..aa8b5fc5f --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r2-paged-b.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+6.695 + batch steps: occupancy=1.00 decode_step_ms=52.72 (predicted_seconds/steps=52.72 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+65.376 + batch steps: occupancy=4.00 decode_step_ms=128.69 (predicted_seconds/steps=514.77 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+454.609 + batch steps: occupancy=8.00 decode_step_ms=447.45 (predicted_seconds/steps=3579.60 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 4079.5 4079.5 19.0 11.9 + 4 4 0 318.0 398.8 7.8 30.7 + 8 8 0 419.3 612.3 2.2 17.9 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r3-dense.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r3-dense.txt new file mode 100644 index 000000000..97a671c4c --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r3-dense.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+6.800 + batch steps: occupancy=1.00 decode_step_ms=53.54 (predicted_seconds/steps=53.54 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+64.412 + batch steps: occupancy=4.00 decode_step_ms=126.80 (predicted_seconds/steps=507.18 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+467.861 + batch steps: occupancy=8.00 decode_step_ms=460.49 (predicted_seconds/steps=3683.94 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 4048.7 4048.7 18.7 11.8 + 4 4 0 321.1 403.3 7.9 31.2 + 8 8 0 435.7 635.0 2.2 17.4 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r3-paged-a.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r3-paged-a.txt new file mode 100644 index 000000000..223541f03 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r3-paged-a.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+6.774 + batch steps: occupancy=1.00 decode_step_ms=53.34 (predicted_seconds/steps=53.34 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+64.776 + batch steps: occupancy=4.00 decode_step_ms=127.51 (predicted_seconds/steps=510.05 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+458.160 + batch steps: occupancy=8.00 decode_step_ms=450.94 (predicted_seconds/steps=3607.56 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 4191.9 4191.9 18.7 11.7 + 4 4 0 325.4 410.5 7.8 31.0 + 8 8 0 434.7 632.5 2.2 17.7 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r3-paged-b.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r3-paged-b.txt new file mode 100644 index 000000000..7288bc3ca --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r3-paged-b.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+6.703 + batch steps: occupancy=1.00 decode_step_ms=52.78 (predicted_seconds/steps=52.78 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+64.967 + batch steps: occupancy=4.00 decode_step_ms=127.89 (predicted_seconds/steps=511.55 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+464.343 + batch steps: occupancy=8.00 decode_step_ms=457.03 (predicted_seconds/steps=3656.24 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 4089.8 4089.8 18.9 11.9 + 4 4 0 318.7 401.8 7.8 30.9 + 8 8 0 425.2 616.8 2.2 17.5 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r4-dense.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r4-dense.txt new file mode 100644 index 000000000..647275273 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r4-dense.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+6.716 + batch steps: occupancy=1.00 decode_step_ms=52.88 (predicted_seconds/steps=52.88 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+65.383 + batch steps: occupancy=4.00 decode_step_ms=128.71 (predicted_seconds/steps=514.83 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+455.357 + batch steps: occupancy=8.00 decode_step_ms=448.19 (predicted_seconds/steps=3585.49 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 4014.9 4014.9 18.9 11.9 + 4 4 0 320.8 404.0 7.8 30.7 + 8 8 0 435.9 630.9 2.2 17.9 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r4-paged-a.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r4-paged-a.txt new file mode 100644 index 000000000..4aab2a373 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r4-paged-a.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+6.736 + batch steps: occupancy=1.00 decode_step_ms=53.04 (predicted_seconds/steps=53.04 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+64.859 + batch steps: occupancy=4.00 decode_step_ms=127.68 (predicted_seconds/steps=510.70 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+455.798 + batch steps: occupancy=8.00 decode_step_ms=448.62 (predicted_seconds/steps=3588.96 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 4221.0 4221.0 18.9 11.7 + 4 4 0 319.6 401.1 7.8 31.0 + 8 8 0 424.8 617.1 2.2 17.8 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r4-paged-b.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r4-paged-b.txt new file mode 100644 index 000000000..e286022f3 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r4-paged-b.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+6.705 + batch steps: occupancy=1.00 decode_step_ms=52.80 (predicted_seconds/steps=52.80 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+65.391 + batch steps: occupancy=4.00 decode_step_ms=128.72 (predicted_seconds/steps=514.89 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+458.411 + batch steps: occupancy=8.00 decode_step_ms=451.19 (predicted_seconds/steps=3609.54 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 4134.9 4134.9 18.9 11.8 + 4 4 0 317.0 398.0 7.8 30.7 + 8 8 0 432.4 628.3 2.2 17.7 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r5-dense.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r5-dense.txt new file mode 100644 index 000000000..c93207b45 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r5-dense.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+6.717 + batch steps: occupancy=1.00 decode_step_ms=52.89 (predicted_seconds/steps=52.89 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+63.908 + batch steps: occupancy=4.00 decode_step_ms=125.80 (predicted_seconds/steps=503.21 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+457.680 + batch steps: occupancy=8.00 decode_step_ms=450.47 (predicted_seconds/steps=3603.78 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 4123.5 4123.5 18.9 11.8 + 4 4 0 318.5 401.4 7.9 31.4 + 8 8 0 435.9 634.9 2.2 17.8 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r5-paged-a.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r5-paged-a.txt new file mode 100644 index 000000000..6991323e8 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r5-paged-a.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+6.747 + batch steps: occupancy=1.00 decode_step_ms=53.13 (predicted_seconds/steps=53.13 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+65.696 + batch steps: occupancy=4.00 decode_step_ms=129.32 (predicted_seconds/steps=517.29 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+457.518 + batch steps: occupancy=8.00 decode_step_ms=450.31 (predicted_seconds/steps=3602.50 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 4132.5 4132.5 18.8 11.8 + 4 4 0 325.3 409.5 7.7 30.6 + 8 8 0 426.6 621.1 2.2 17.8 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r5-paged-b.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r5-paged-b.txt new file mode 100644 index 000000000..13bab1744 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/client-llama4-p512-r5-paged-b.txt @@ -0,0 +1,30 @@ +Server: http://127.0.0.1:18080 +Model: llama-4-scout-17b-16e-instruct-4bit +Prompt: ~512 tokens (2591 chars), style plain +Max tokens: 128 +Concurrency: [1, 4, 8] + +>>> concurrency=1 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+127 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+6.705 + batch steps: occupancy=1.00 decode_step_ms=52.80 (predicted_seconds/steps=52.80 ms) + +>>> concurrency=4 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+508 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+66.159 + batch steps: occupancy=4.00 decode_step_ms=130.23 (predicted_seconds/steps=520.94 ms) + +>>> concurrency=8 ... + decode paths: paged_decode_launches_total{path="fused_v2"}=+0 paged_decode_launches_total{path="gather"}=+0 paged_decode_launches_total{path="cascade"}=+0 cascade_shared_tokens_total=+0 cascade_member_sequences_total=+0 cascade_failures_total=+0 prompt_cache_hits_total=+0 + NOTE: no cascade launch in this level. Whatever this level measured, it was not cascade attention. + batch counters: decode_steps=+127 decode_tokens=+1016 mixed_steps=+0 prefill_chunks=+0 tokens_predicted_seconds=+452.879 + batch steps: occupancy=8.00 decode_step_ms=445.75 (predicted_seconds/steps=3565.98 ms) + +conc ok fail ttft_ms(mean) ttft_ms(p95) decode_tok_s(mean) aggregate_tok_s +--------------------------------------------------------------------------------- + 1 1 0 4187.4 4187.4 18.9 11.7 + 4 4 0 318.0 401.3 7.7 30.4 + 8 8 0 441.5 640.6 2.2 17.9 + diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/complete b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/complete new file mode 100644 index 000000000..e69de29bb diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/environment.txt b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/environment.txt new file mode 100644 index 000000000..0aa4d611f --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/environment.txt @@ -0,0 +1,20 @@ +started_utc=2026-10-11T04:57:35Z +worktree=/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230 +worktree_clean=true +binary=/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/target/release/mlxcel +binary_sha256=0cc49217021dab12b5add25dc2bb8de7e90c405c0b406607b244a8060d3ed800 +binary_size_bytes=108106312 +binary_mtime_utc=2026-10-11T03:25:49Z +commit=35da2dbe32c731ff6645b2b32a2117af2965f332 +mlx_cmake_git_tag=81ba1c6a0e50a9268b931579c2d4f1158b9aab5a +mlx_checkout=/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/target/release/build/mlxcel-core-4c8821fbbcc3d8f4/out/build/_deps/mlx-src +mlx_checkout_commit=81ba1c6a0e50a9268b931579c2d4f1158b9aab5a +hostgate_source=/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/docs/benchmark_results/data/sdpa-plan-bucket-gb10-2026-09-12/harness/hostgate.py +hostgate_sha256=69e6d3f468c42ba7e478578e3bde9efb9c5a30165866d8e9da19440a6eb9413f +gate_sha256=15a50c2718c3134750bf57e3195d294ea783cd7ae2f42940f6be1ad2dd63a607 +analyzer_sha256=f17233d3e5b5339424c050404ea81a171ad43b0abd74ebb4d4ddc681f894f3f8 +run_sha256=3b5816d50ae7cb4e5d5a5425cd31cadbcb05f33b57111c91f8e4edf35f994854 +host=spark-102 +uname=Linux spark-102 7.0.0-1019-nvidia #19~24.04.2-Ubuntu SMP PREEMPT_DYNAMIC Sun Sep 6 18:35:11 UTC 2 aarch64 aarch64 aarch64 GNU/Linux +gpu=NVIDIA GB10, 580.178.04, 0000000F:01:00.0 +completed_utc=2026-10-11T07:12:28Z diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/hostgate.jsonl b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/hostgate.jsonl new file mode 100644 index 000000000..af11b8df9 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/hostgate.jsonl @@ -0,0 +1,61 @@ +{"completed_utc": "2026-10-11T04:57:36Z", "label": "run-preflight", "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "started_utc": "2026-10-11T04:57:36Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T04:58:33Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p512-r1-paged-a", "load1": 0.17578125, "mem_available_gib": 112.22350311279297, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T04:57:36Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T04:59:46Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p512-r1-dense", "load1": 0.19189453125, "mem_available_gib": 112.24962615966797, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T04:58:49Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:00:58Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p512-r1-paged-b", "load1": 0.1904296875, "mem_available_gib": 112.25072860717773, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:00:02Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:02:10Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p512-r2-dense", "load1": 0.32470703125, "mem_available_gib": 112.17797470092773, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:01:14Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:03:23Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p512-r2-paged-b", "load1": 0.365234375, "mem_available_gib": 112.21145248413086, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:02:26Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:04:35Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p512-r2-paged-a", "load1": 0.2490234375, "mem_available_gib": 112.14068603515625, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:03:39Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:05:48Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p512-r3-paged-b", "load1": 0.2880859375, "mem_available_gib": 112.1058349609375, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:04:51Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:07:00Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p512-r3-paged-a", "load1": 0.36474609375, "mem_available_gib": 112.16807556152344, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:06:04Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:08:13Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p512-r3-dense", "load1": 0.66357421875, "mem_available_gib": 112.22763061523438, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:07:16Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:09:25Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p512-r4-paged-a", "load1": 3.25732421875, "mem_available_gib": 112.2436637878418, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:08:29Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:10:38Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p512-r4-dense", "load1": 3.06982421875, "mem_available_gib": 112.20537185668945, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:09:41Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:11:50Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p512-r4-paged-b", "load1": 2.357421875, "mem_available_gib": 112.2288589477539, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:10:53Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:13:02Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p512-r5-dense", "load1": 1.8623046875, "mem_available_gib": 112.13740921020508, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:12:06Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:14:15Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p512-r5-paged-b", "load1": 1.46435546875, "mem_available_gib": 112.07698440551758, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:13:18Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:15:27Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p512-r5-paged-a", "load1": 1.0947265625, "mem_available_gib": 112.17715072631836, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:14:31Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:16:40Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p4096-r1-paged-a", "load1": 0.8740234375, "mem_available_gib": 112.11929702758789, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:15:43Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:17:54Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p4096-r1-dense", "load1": 0.9892578125, "mem_available_gib": 112.11450958251953, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:16:57Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:19:08Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p4096-r1-paged-b", "load1": 0.86669921875, "mem_available_gib": 112.21323776245117, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:18:11Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:20:22Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p4096-r2-dense", "load1": 1.09130859375, "mem_available_gib": 112.21668243408203, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:19:25Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:21:36Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p4096-r2-paged-b", "load1": 1.00537109375, "mem_available_gib": 112.2269515991211, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:20:39Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:22:50Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p4096-r2-paged-a", "load1": 0.71826171875, "mem_available_gib": 112.16401290893555, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:21:53Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:24:04Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p4096-r3-paged-b", "load1": 0.55810546875, "mem_available_gib": 112.11655807495117, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:23:07Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:25:17Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p4096-r3-paged-a", "load1": 0.322265625, "mem_available_gib": 112.10714721679688, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:24:21Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:26:31Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p4096-r3-dense", "load1": 0.349609375, "mem_available_gib": 112.08111572265625, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:25:35Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:27:45Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p4096-r4-paged-a", "load1": 0.33447265625, "mem_available_gib": 112.15883255004883, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:26:49Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:28:59Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p4096-r4-dense", "load1": 0.2373046875, "mem_available_gib": 112.18478393554688, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:28:03Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:30:13Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p4096-r4-paged-b", "load1": 0.27392578125, "mem_available_gib": 112.14545822143555, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:29:17Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:31:27Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p4096-r5-dense", "load1": 0.53857421875, "mem_available_gib": 112.14363098144531, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:30:30Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:32:41Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p4096-r5-paged-b", "load1": 0.3798828125, "mem_available_gib": 112.19094848632812, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:31:44Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:33:55Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "gemma3-p4096-r5-paged-a", "load1": 0.86962890625, "mem_available_gib": 112.07434463500977, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:32:59Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:35:09Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p512-r1-paged-a", "load1": 0.427734375, "mem_available_gib": 112.16619110107422, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 6, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:34:13Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:38:32Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p512-r1-dense", "load1": 0.6328125, "mem_available_gib": 112.58162689208984, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 7, "nvrm_window_5m": 1, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:37:35Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:41:36Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p512-r1-paged-b", "load1": 0.45654296875, "mem_available_gib": 112.51552963256836, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 7, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:40:39Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:44:40Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p512-r2-dense", "load1": 0.62060546875, "mem_available_gib": 110.41999816894531, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 9, "nvrm_window_5m": 2, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:43:43Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:47:44Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p512-r2-paged-b", "load1": 0.56201171875, "mem_available_gib": 112.36882781982422, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 10, "nvrm_window_5m": 1, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:46:48Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:50:47Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p512-r2-paged-a", "load1": 0.64208984375, "mem_available_gib": 111.19093704223633, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 13, "nvrm_window_5m": 3, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:49:51Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:53:51Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p512-r3-paged-b", "load1": 0.5595703125, "mem_available_gib": 110.60594177246094, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 15, "nvrm_window_5m": 2, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:52:54Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T05:56:56Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p512-r3-paged-a", "load1": 0.7529296875, "mem_available_gib": 110.55312728881836, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 15, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:55:59Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T06:00:00Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p512-r3-dense", "load1": 0.72900390625, "mem_available_gib": 112.57743453979492, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 17, "nvrm_window_5m": 2, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T05:59:04Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T06:03:06Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p512-r4-paged-a", "load1": 0.3447265625, "mem_available_gib": 109.87887573242188, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 17, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T06:02:09Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T06:06:08Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p512-r4-dense", "load1": 0.57666015625, "mem_available_gib": 109.71785736083984, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 18, "nvrm_window_5m": 1, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T06:05:11Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T06:09:12Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p512-r4-paged-b", "load1": 0.39453125, "mem_available_gib": 112.45189666748047, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 20, "nvrm_window_5m": 2, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T06:08:15Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T06:12:17Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p512-r5-dense", "load1": 0.50927734375, "mem_available_gib": 111.34344482421875, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 20, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T06:11:21Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T06:15:21Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p512-r5-paged-b", "load1": 0.642578125, "mem_available_gib": 112.4449577331543, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 25, "nvrm_window_5m": 5, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T06:14:25Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T06:18:24Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p512-r5-paged-a", "load1": 0.375, "mem_available_gib": 112.52336502075195, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 28, "nvrm_window_5m": 3, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T06:17:28Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T06:21:29Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p4096-r1-paged-a", "load1": 0.3046875, "mem_available_gib": 112.36130142211914, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 30, "nvrm_window_5m": 2, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T06:20:32Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T06:24:59Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p4096-r1-dense", "load1": 0.29150390625, "mem_available_gib": 112.61242294311523, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 31, "nvrm_window_5m": 1, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T06:24:02Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T06:28:25Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p4096-r1-paged-b", "load1": 0.50146484375, "mem_available_gib": 112.7584114074707, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 36, "nvrm_window_5m": 5, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T06:27:29Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T06:31:53Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p4096-r2-dense", "load1": 0.42236328125, "mem_available_gib": 112.69794845581055, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 36, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T06:30:57Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T06:35:20Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p4096-r2-paged-b", "load1": 0.5068359375, "mem_available_gib": 112.68205642700195, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 36, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T06:34:23Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T06:38:49Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p4096-r2-paged-a", "load1": 0.525390625, "mem_available_gib": 112.68825912475586, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 40, "nvrm_window_5m": 4, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T06:37:52Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T06:42:18Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p4096-r3-paged-b", "load1": 0.5234375, "mem_available_gib": 112.52041244506836, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 40, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T06:41:21Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T06:45:45Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p4096-r3-paged-a", "load1": 0.44140625, "mem_available_gib": 112.65090560913086, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 40, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T06:44:49Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T06:49:15Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p4096-r3-dense", "load1": 0.484375, "mem_available_gib": 112.44796752929688, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 41, "nvrm_window_5m": 1, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T06:48:18Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T06:52:42Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p4096-r4-paged-a", "load1": 0.50244140625, "mem_available_gib": 112.57556915283203, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 41, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T06:51:46Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T06:56:09Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p4096-r4-dense", "load1": 0.47802734375, "mem_available_gib": 112.62853622436523, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 46, "nvrm_window_5m": 5, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T06:55:13Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T06:59:35Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p4096-r4-paged-b", "load1": 0.4873046875, "mem_available_gib": 110.36702346801758, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 46, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T06:58:38Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T07:03:01Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p4096-r5-dense", "load1": 0.46923828125, "mem_available_gib": 112.68459701538086, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 46, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T07:02:04Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T07:06:28Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p4096-r5-paged-b", "load1": 0.5361328125, "mem_available_gib": 112.65660858154297, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 46, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T07:05:32Z"} +{"accepted_quiet": true, "busy_procs": [], "ci_job_running": false, "completed_utc": "2026-10-11T07:09:57Z", "driver_wait_s": 0, "foreign_model_procs": [], "gate_wait_s": 0, "label": "llama4-p4096-r5-paged-a", "load1": 0.46533203125, "mem_available_gib": 112.62932586669922, "mem_floor_gib": 32.0, "mem_wait_s": 0, "nvrm_total_max": 400, "nvrm_total_since_boot": 46, "nvrm_window_5m": 0, "nvrm_window_max": 50, "port": 18080, "port_free_after_wait": true, "port_free_before_wait": true, "quiet_rechecks": 0, "started_utc": "2026-10-11T07:09:00Z"} diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/run.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/run.log new file mode 100644 index 000000000..dd2db35a5 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/run.log @@ -0,0 +1,181 @@ +GATE gemma3-p512-r1-paged-a 2026-10-11T04:57:36Z +START gemma3-p512-r1-paged-a 2026-10-11T04:58:33Z +DONE gemma3-p512-r1-paged-a 2026-10-11T04:58:49Z +GATE gemma3-p512-r1-dense 2026-10-11T04:58:49Z +START gemma3-p512-r1-dense 2026-10-11T04:59:46Z +DONE gemma3-p512-r1-dense 2026-10-11T05:00:02Z +GATE gemma3-p512-r1-paged-b 2026-10-11T05:00:02Z +START gemma3-p512-r1-paged-b 2026-10-11T05:00:58Z +DONE gemma3-p512-r1-paged-b 2026-10-11T05:01:14Z +GATE gemma3-p512-r2-dense 2026-10-11T05:01:14Z +START gemma3-p512-r2-dense 2026-10-11T05:02:10Z +DONE gemma3-p512-r2-dense 2026-10-11T05:02:26Z +GATE gemma3-p512-r2-paged-b 2026-10-11T05:02:26Z +START gemma3-p512-r2-paged-b 2026-10-11T05:03:23Z +DONE gemma3-p512-r2-paged-b 2026-10-11T05:03:39Z +GATE gemma3-p512-r2-paged-a 2026-10-11T05:03:39Z +START gemma3-p512-r2-paged-a 2026-10-11T05:04:35Z +DONE gemma3-p512-r2-paged-a 2026-10-11T05:04:51Z +GATE gemma3-p512-r3-paged-b 2026-10-11T05:04:51Z +START gemma3-p512-r3-paged-b 2026-10-11T05:05:48Z +DONE gemma3-p512-r3-paged-b 2026-10-11T05:06:04Z +GATE gemma3-p512-r3-paged-a 2026-10-11T05:06:04Z +START gemma3-p512-r3-paged-a 2026-10-11T05:07:00Z +DONE gemma3-p512-r3-paged-a 2026-10-11T05:07:16Z +GATE gemma3-p512-r3-dense 2026-10-11T05:07:16Z +START gemma3-p512-r3-dense 2026-10-11T05:08:13Z +DONE gemma3-p512-r3-dense 2026-10-11T05:08:29Z +GATE gemma3-p512-r4-paged-a 2026-10-11T05:08:29Z +START gemma3-p512-r4-paged-a 2026-10-11T05:09:25Z +DONE gemma3-p512-r4-paged-a 2026-10-11T05:09:41Z +GATE gemma3-p512-r4-dense 2026-10-11T05:09:41Z +START gemma3-p512-r4-dense 2026-10-11T05:10:38Z +DONE gemma3-p512-r4-dense 2026-10-11T05:10:53Z +GATE gemma3-p512-r4-paged-b 2026-10-11T05:10:53Z +START gemma3-p512-r4-paged-b 2026-10-11T05:11:50Z +DONE gemma3-p512-r4-paged-b 2026-10-11T05:12:06Z +GATE gemma3-p512-r5-dense 2026-10-11T05:12:06Z +START gemma3-p512-r5-dense 2026-10-11T05:13:02Z +DONE gemma3-p512-r5-dense 2026-10-11T05:13:18Z +GATE gemma3-p512-r5-paged-b 2026-10-11T05:13:18Z +START gemma3-p512-r5-paged-b 2026-10-11T05:14:15Z +DONE gemma3-p512-r5-paged-b 2026-10-11T05:14:31Z +GATE gemma3-p512-r5-paged-a 2026-10-11T05:14:31Z +START gemma3-p512-r5-paged-a 2026-10-11T05:15:27Z +DONE gemma3-p512-r5-paged-a 2026-10-11T05:15:43Z +GATE gemma3-p4096-r1-paged-a 2026-10-11T05:15:43Z +START gemma3-p4096-r1-paged-a 2026-10-11T05:16:40Z +DONE gemma3-p4096-r1-paged-a 2026-10-11T05:16:57Z +GATE gemma3-p4096-r1-dense 2026-10-11T05:16:57Z +START gemma3-p4096-r1-dense 2026-10-11T05:17:54Z +DONE gemma3-p4096-r1-dense 2026-10-11T05:18:11Z +GATE gemma3-p4096-r1-paged-b 2026-10-11T05:18:11Z +START gemma3-p4096-r1-paged-b 2026-10-11T05:19:08Z +DONE gemma3-p4096-r1-paged-b 2026-10-11T05:19:25Z +GATE gemma3-p4096-r2-dense 2026-10-11T05:19:25Z +START gemma3-p4096-r2-dense 2026-10-11T05:20:22Z +DONE gemma3-p4096-r2-dense 2026-10-11T05:20:39Z +GATE gemma3-p4096-r2-paged-b 2026-10-11T05:20:39Z +START gemma3-p4096-r2-paged-b 2026-10-11T05:21:36Z +DONE gemma3-p4096-r2-paged-b 2026-10-11T05:21:53Z +GATE gemma3-p4096-r2-paged-a 2026-10-11T05:21:53Z +START gemma3-p4096-r2-paged-a 2026-10-11T05:22:50Z +DONE gemma3-p4096-r2-paged-a 2026-10-11T05:23:07Z +GATE gemma3-p4096-r3-paged-b 2026-10-11T05:23:07Z +START gemma3-p4096-r3-paged-b 2026-10-11T05:24:04Z +DONE gemma3-p4096-r3-paged-b 2026-10-11T05:24:21Z +GATE gemma3-p4096-r3-paged-a 2026-10-11T05:24:21Z +START gemma3-p4096-r3-paged-a 2026-10-11T05:25:17Z +DONE gemma3-p4096-r3-paged-a 2026-10-11T05:25:35Z +GATE gemma3-p4096-r3-dense 2026-10-11T05:25:35Z +START gemma3-p4096-r3-dense 2026-10-11T05:26:31Z +DONE gemma3-p4096-r3-dense 2026-10-11T05:26:49Z +GATE gemma3-p4096-r4-paged-a 2026-10-11T05:26:49Z +START gemma3-p4096-r4-paged-a 2026-10-11T05:27:45Z +DONE gemma3-p4096-r4-paged-a 2026-10-11T05:28:03Z +GATE gemma3-p4096-r4-dense 2026-10-11T05:28:03Z +START gemma3-p4096-r4-dense 2026-10-11T05:28:59Z +DONE gemma3-p4096-r4-dense 2026-10-11T05:29:17Z +GATE gemma3-p4096-r4-paged-b 2026-10-11T05:29:17Z +START gemma3-p4096-r4-paged-b 2026-10-11T05:30:13Z +DONE gemma3-p4096-r4-paged-b 2026-10-11T05:30:30Z +GATE gemma3-p4096-r5-dense 2026-10-11T05:30:30Z +START gemma3-p4096-r5-dense 2026-10-11T05:31:27Z +DONE gemma3-p4096-r5-dense 2026-10-11T05:31:44Z +GATE gemma3-p4096-r5-paged-b 2026-10-11T05:31:44Z +START gemma3-p4096-r5-paged-b 2026-10-11T05:32:41Z +DONE gemma3-p4096-r5-paged-b 2026-10-11T05:32:59Z +GATE gemma3-p4096-r5-paged-a 2026-10-11T05:32:59Z +START gemma3-p4096-r5-paged-a 2026-10-11T05:33:55Z +DONE gemma3-p4096-r5-paged-a 2026-10-11T05:34:13Z +GATE llama4-p512-r1-paged-a 2026-10-11T05:34:13Z +START llama4-p512-r1-paged-a 2026-10-11T05:35:09Z +DONE llama4-p512-r1-paged-a 2026-10-11T05:37:35Z +GATE llama4-p512-r1-dense 2026-10-11T05:37:35Z +START llama4-p512-r1-dense 2026-10-11T05:38:32Z +DONE llama4-p512-r1-dense 2026-10-11T05:40:39Z +GATE llama4-p512-r1-paged-b 2026-10-11T05:40:39Z +START llama4-p512-r1-paged-b 2026-10-11T05:41:36Z +DONE llama4-p512-r1-paged-b 2026-10-11T05:43:43Z +GATE llama4-p512-r2-dense 2026-10-11T05:43:43Z +START llama4-p512-r2-dense 2026-10-11T05:44:40Z +DONE llama4-p512-r2-dense 2026-10-11T05:46:48Z +GATE llama4-p512-r2-paged-b 2026-10-11T05:46:48Z +START llama4-p512-r2-paged-b 2026-10-11T05:47:44Z +DONE llama4-p512-r2-paged-b 2026-10-11T05:49:51Z +GATE llama4-p512-r2-paged-a 2026-10-11T05:49:51Z +START llama4-p512-r2-paged-a 2026-10-11T05:50:47Z +DONE llama4-p512-r2-paged-a 2026-10-11T05:52:54Z +GATE llama4-p512-r3-paged-b 2026-10-11T05:52:54Z +START llama4-p512-r3-paged-b 2026-10-11T05:53:51Z +DONE llama4-p512-r3-paged-b 2026-10-11T05:55:59Z +GATE llama4-p512-r3-paged-a 2026-10-11T05:55:59Z +START llama4-p512-r3-paged-a 2026-10-11T05:56:56Z +DONE llama4-p512-r3-paged-a 2026-10-11T05:59:04Z +GATE llama4-p512-r3-dense 2026-10-11T05:59:04Z +START llama4-p512-r3-dense 2026-10-11T06:00:00Z +DONE llama4-p512-r3-dense 2026-10-11T06:02:09Z +GATE llama4-p512-r4-paged-a 2026-10-11T06:02:09Z +START llama4-p512-r4-paged-a 2026-10-11T06:03:06Z +DONE llama4-p512-r4-paged-a 2026-10-11T06:05:11Z +GATE llama4-p512-r4-dense 2026-10-11T06:05:11Z +START llama4-p512-r4-dense 2026-10-11T06:06:08Z +DONE llama4-p512-r4-dense 2026-10-11T06:08:15Z +GATE llama4-p512-r4-paged-b 2026-10-11T06:08:15Z +START llama4-p512-r4-paged-b 2026-10-11T06:09:12Z +DONE llama4-p512-r4-paged-b 2026-10-11T06:11:21Z +GATE llama4-p512-r5-dense 2026-10-11T06:11:21Z +START llama4-p512-r5-dense 2026-10-11T06:12:17Z +DONE llama4-p512-r5-dense 2026-10-11T06:14:25Z +GATE llama4-p512-r5-paged-b 2026-10-11T06:14:25Z +START llama4-p512-r5-paged-b 2026-10-11T06:15:21Z +DONE llama4-p512-r5-paged-b 2026-10-11T06:17:28Z +GATE llama4-p512-r5-paged-a 2026-10-11T06:17:28Z +START llama4-p512-r5-paged-a 2026-10-11T06:18:24Z +DONE llama4-p512-r5-paged-a 2026-10-11T06:20:32Z +GATE llama4-p4096-r1-paged-a 2026-10-11T06:20:32Z +START llama4-p4096-r1-paged-a 2026-10-11T06:21:29Z +DONE llama4-p4096-r1-paged-a 2026-10-11T06:24:02Z +GATE llama4-p4096-r1-dense 2026-10-11T06:24:02Z +START llama4-p4096-r1-dense 2026-10-11T06:24:59Z +DONE llama4-p4096-r1-dense 2026-10-11T06:27:29Z +GATE llama4-p4096-r1-paged-b 2026-10-11T06:27:29Z +START llama4-p4096-r1-paged-b 2026-10-11T06:28:25Z +DONE llama4-p4096-r1-paged-b 2026-10-11T06:30:57Z +GATE llama4-p4096-r2-dense 2026-10-11T06:30:57Z +START llama4-p4096-r2-dense 2026-10-11T06:31:53Z +DONE llama4-p4096-r2-dense 2026-10-11T06:34:23Z +GATE llama4-p4096-r2-paged-b 2026-10-11T06:34:23Z +START llama4-p4096-r2-paged-b 2026-10-11T06:35:20Z +DONE llama4-p4096-r2-paged-b 2026-10-11T06:37:52Z +GATE llama4-p4096-r2-paged-a 2026-10-11T06:37:52Z +START llama4-p4096-r2-paged-a 2026-10-11T06:38:49Z +DONE llama4-p4096-r2-paged-a 2026-10-11T06:41:21Z +GATE llama4-p4096-r3-paged-b 2026-10-11T06:41:21Z +START llama4-p4096-r3-paged-b 2026-10-11T06:42:18Z +DONE llama4-p4096-r3-paged-b 2026-10-11T06:44:49Z +GATE llama4-p4096-r3-paged-a 2026-10-11T06:44:49Z +START llama4-p4096-r3-paged-a 2026-10-11T06:45:45Z +DONE llama4-p4096-r3-paged-a 2026-10-11T06:48:18Z +GATE llama4-p4096-r3-dense 2026-10-11T06:48:18Z +START llama4-p4096-r3-dense 2026-10-11T06:49:15Z +DONE llama4-p4096-r3-dense 2026-10-11T06:51:46Z +GATE llama4-p4096-r4-paged-a 2026-10-11T06:51:46Z +START llama4-p4096-r4-paged-a 2026-10-11T06:52:42Z +DONE llama4-p4096-r4-paged-a 2026-10-11T06:55:13Z +GATE llama4-p4096-r4-dense 2026-10-11T06:55:13Z +START llama4-p4096-r4-dense 2026-10-11T06:56:09Z +DONE llama4-p4096-r4-dense 2026-10-11T06:58:38Z +GATE llama4-p4096-r4-paged-b 2026-10-11T06:58:38Z +START llama4-p4096-r4-paged-b 2026-10-11T06:59:35Z +DONE llama4-p4096-r4-paged-b 2026-10-11T07:02:04Z +GATE llama4-p4096-r5-dense 2026-10-11T07:02:04Z +START llama4-p4096-r5-dense 2026-10-11T07:03:01Z +DONE llama4-p4096-r5-dense 2026-10-11T07:05:32Z +GATE llama4-p4096-r5-paged-b 2026-10-11T07:05:32Z +START llama4-p4096-r5-paged-b 2026-10-11T07:06:28Z +DONE llama4-p4096-r5-paged-b 2026-10-11T07:09:00Z +GATE llama4-p4096-r5-paged-a 2026-10-11T07:09:00Z +START llama4-p4096-r5-paged-a 2026-10-11T07:09:57Z +DONE llama4-p4096-r5-paged-a 2026-10-11T07:12:28Z +ALLDONE 2026-10-11T07:12:28Z diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/schedule.tsv b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/schedule.tsv new file mode 100644 index 000000000..4c4c980e9 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/schedule.tsv @@ -0,0 +1,61 @@ +model prompt_tokens round position arm backend started_utc +gemma3 512 1 1 paged-a paged 2026-10-11T04:58:33Z +gemma3 512 1 2 dense dense 2026-10-11T04:59:46Z +gemma3 512 1 3 paged-b paged 2026-10-11T05:00:58Z +gemma3 512 2 1 dense dense 2026-10-11T05:02:10Z +gemma3 512 2 2 paged-b paged 2026-10-11T05:03:23Z +gemma3 512 2 3 paged-a paged 2026-10-11T05:04:35Z +gemma3 512 3 1 paged-b paged 2026-10-11T05:05:48Z +gemma3 512 3 2 paged-a paged 2026-10-11T05:07:00Z +gemma3 512 3 3 dense dense 2026-10-11T05:08:13Z +gemma3 512 4 1 paged-a paged 2026-10-11T05:09:25Z +gemma3 512 4 2 dense dense 2026-10-11T05:10:38Z +gemma3 512 4 3 paged-b paged 2026-10-11T05:11:50Z +gemma3 512 5 1 dense dense 2026-10-11T05:13:02Z +gemma3 512 5 2 paged-b paged 2026-10-11T05:14:15Z +gemma3 512 5 3 paged-a paged 2026-10-11T05:15:27Z +gemma3 4096 1 1 paged-a paged 2026-10-11T05:16:40Z +gemma3 4096 1 2 dense dense 2026-10-11T05:17:54Z +gemma3 4096 1 3 paged-b paged 2026-10-11T05:19:08Z +gemma3 4096 2 1 dense dense 2026-10-11T05:20:22Z +gemma3 4096 2 2 paged-b paged 2026-10-11T05:21:36Z +gemma3 4096 2 3 paged-a paged 2026-10-11T05:22:50Z +gemma3 4096 3 1 paged-b paged 2026-10-11T05:24:04Z +gemma3 4096 3 2 paged-a paged 2026-10-11T05:25:17Z +gemma3 4096 3 3 dense dense 2026-10-11T05:26:31Z +gemma3 4096 4 1 paged-a paged 2026-10-11T05:27:45Z +gemma3 4096 4 2 dense dense 2026-10-11T05:28:59Z +gemma3 4096 4 3 paged-b paged 2026-10-11T05:30:13Z +gemma3 4096 5 1 dense dense 2026-10-11T05:31:27Z +gemma3 4096 5 2 paged-b paged 2026-10-11T05:32:41Z +gemma3 4096 5 3 paged-a paged 2026-10-11T05:33:55Z +llama4 512 1 1 paged-a paged 2026-10-11T05:35:09Z +llama4 512 1 2 dense dense 2026-10-11T05:38:32Z +llama4 512 1 3 paged-b paged 2026-10-11T05:41:36Z +llama4 512 2 1 dense dense 2026-10-11T05:44:40Z +llama4 512 2 2 paged-b paged 2026-10-11T05:47:44Z +llama4 512 2 3 paged-a paged 2026-10-11T05:50:47Z +llama4 512 3 1 paged-b paged 2026-10-11T05:53:51Z +llama4 512 3 2 paged-a paged 2026-10-11T05:56:56Z +llama4 512 3 3 dense dense 2026-10-11T06:00:00Z +llama4 512 4 1 paged-a paged 2026-10-11T06:03:06Z +llama4 512 4 2 dense dense 2026-10-11T06:06:08Z +llama4 512 4 3 paged-b paged 2026-10-11T06:09:12Z +llama4 512 5 1 dense dense 2026-10-11T06:12:17Z +llama4 512 5 2 paged-b paged 2026-10-11T06:15:21Z +llama4 512 5 3 paged-a paged 2026-10-11T06:18:24Z +llama4 4096 1 1 paged-a paged 2026-10-11T06:21:29Z +llama4 4096 1 2 dense dense 2026-10-11T06:24:59Z +llama4 4096 1 3 paged-b paged 2026-10-11T06:28:25Z +llama4 4096 2 1 dense dense 2026-10-11T06:31:53Z +llama4 4096 2 2 paged-b paged 2026-10-11T06:35:20Z +llama4 4096 2 3 paged-a paged 2026-10-11T06:38:49Z +llama4 4096 3 1 paged-b paged 2026-10-11T06:42:18Z +llama4 4096 3 2 paged-a paged 2026-10-11T06:45:45Z +llama4 4096 3 3 dense dense 2026-10-11T06:49:15Z +llama4 4096 4 1 paged-a paged 2026-10-11T06:52:42Z +llama4 4096 4 2 dense dense 2026-10-11T06:56:09Z +llama4 4096 4 3 paged-b paged 2026-10-11T06:59:35Z +llama4 4096 5 1 dense dense 2026-10-11T07:03:01Z +llama4 4096 5 2 paged-b paged 2026-10-11T07:06:28Z +llama4 4096 5 3 paged-a paged 2026-10-11T07:09:57Z diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r1-dense.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r1-dense.log new file mode 100644 index 000000000..15b861fea --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r1-dense.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=dense +2026-10-11T05:17:54.634571Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:17:54.634601Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:17:54.634706Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:17:54.634770Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:17:54.634772Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:17:54.634774Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:17:55.213307Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:17:55.213829Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:17:55.214002Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:17:55.788311Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:17:55.788326Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:17:56.388764Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.600s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.600417111 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:17:56.393486Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:17:56.393507Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4) +2026-10-11T05:17:56.393797Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:17:56.393809Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:17:56.393831Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:17:56.394699Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:17:56.809458Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 415ms prompt_tokens=2 cached_tokens=0 generation_time_ms=415 +2026-10-11T05:17:56.809544Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:17:56.810448Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:17:56.810466Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:17:56.810469Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:17:56.810471Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:17:56.810472Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:17:56.810473Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:17:56.810475Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:17:56.810476Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:17:56.810477Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:17:56.810478Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:17:56.810480Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:17:56.810481Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:17:56.810482Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:17:57.365774Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:17:58.485308Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 1113ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=1113 +2026-10-11T05:18:00.890328Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2370ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2370 +2026-10-11T05:18:00.890720Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2371ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2371 +2026-10-11T05:18:00.890982Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2371ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2371 +2026-10-11T05:18:00.891217Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2371ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2371 +2026-10-11T05:18:06.434151Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5509ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5509 +2026-10-11T05:18:06.434738Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5522ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5522 +2026-10-11T05:18:06.435078Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5522ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5522 +2026-10-11T05:18:06.435357Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5510ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5510 +2026-10-11T05:18:06.435615Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5510ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5510 +2026-10-11T05:18:06.435858Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5510ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5510 +2026-10-11T05:18:06.436094Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5511ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5511 +2026-10-11T05:18:06.436337Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5524ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5524 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r1-paged-a.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r1-paged-a.log new file mode 100644 index 000000000..8012548e0 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r1-paged-a.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=paged +2026-10-11T05:16:40.671960Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:16:40.671993Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:16:40.672102Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:16:40.672180Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:16:40.672182Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:16:40.672184Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:16:41.298817Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:16:41.299373Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:16:41.299554Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:16:41.912624Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:16:41.913141Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:16:42.505542Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.592s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.592349395 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:16:42.510021Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:16:42.510035Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T05:16:42.510288Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:16:42.510299Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:16:42.510318Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:16:42.511171Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:16:42.862128Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 351ms prompt_tokens=2 cached_tokens=0 generation_time_ms=351 +2026-10-11T05:16:42.862170Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:16:42.863077Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:16:42.863103Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:16:42.863106Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:16:42.863108Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:16:42.863109Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:16:42.863110Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:16:42.863111Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:16:42.863113Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:16:42.863114Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:16:42.863115Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:16:42.863116Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:16:42.863117Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:16:42.863119Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:16:43.392706Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:16:44.495847Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 1096ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=1096 +2026-10-11T05:16:46.831586Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2294ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2294 +2026-10-11T05:16:46.832130Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2295ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2295 +2026-10-11T05:16:46.832506Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2295ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2295 +2026-10-11T05:16:46.832832Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2295ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2295 +2026-10-11T05:16:52.381589Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5516ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5516 +2026-10-11T05:16:52.382589Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5530ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5530 +2026-10-11T05:16:52.383363Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5518ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5518 +2026-10-11T05:16:52.384095Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5531ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5531 +2026-10-11T05:16:52.384779Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5532ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5532 +2026-10-11T05:16:52.385432Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5520ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5520 +2026-10-11T05:16:52.386081Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5521ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5521 +2026-10-11T05:16:52.386762Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5522ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5522 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r1-paged-b.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r1-paged-b.log new file mode 100644 index 000000000..61e9981a6 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r1-paged-b.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=paged +2026-10-11T05:19:08.657688Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:19:08.657719Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:19:08.657824Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:19:08.657883Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:19:08.657886Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:19:08.657888Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:19:09.238327Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:19:09.238853Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:19:09.239030Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:19:09.813985Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:19:09.814587Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:19:10.433009Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.618s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.618357356 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:19:10.437801Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:19:10.437825Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T05:19:10.438125Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:19:10.438135Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:19:10.438156Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:19:10.439057Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:19:10.844077Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 405ms prompt_tokens=2 cached_tokens=0 generation_time_ms=405 +2026-10-11T05:19:10.844121Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:19:10.845019Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:19:10.845050Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:19:10.845054Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:19:10.845055Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:19:10.845057Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:19:10.845058Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:19:10.845059Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:19:10.845060Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:19:10.845062Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:19:10.845063Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:19:10.845064Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:19:10.845065Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:19:10.845067Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:19:11.408639Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:19:12.530072Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 1114ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=1114 +2026-10-11T05:19:14.872976Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2306ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2306 +2026-10-11T05:19:14.873503Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2307ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2307 +2026-10-11T05:19:14.873903Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2307ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2307 +2026-10-11T05:19:14.874250Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2308ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2308 +2026-10-11T05:19:20.445479Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5550ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5550 +2026-10-11T05:19:20.446162Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5538ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5538 +2026-10-11T05:19:20.446641Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5538ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5538 +2026-10-11T05:19:20.447025Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5539ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5539 +2026-10-11T05:19:20.447366Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5539ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5539 +2026-10-11T05:19:20.447701Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5539ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5539 +2026-10-11T05:19:20.448040Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5540ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5540 +2026-10-11T05:19:20.448374Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5540ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5540 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r2-dense.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r2-dense.log new file mode 100644 index 000000000..503af9e1e --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r2-dense.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=dense +2026-10-11T05:20:22.685870Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:20:22.685903Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:20:22.686012Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:20:22.686071Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:20:22.686074Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:20:22.686077Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:20:23.316466Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:20:23.317047Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:20:23.317233Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:20:23.944910Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:20:23.945499Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:20:24.530295Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.585s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.584766826 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:20:24.534760Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:20:24.534775Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4) +2026-10-11T05:20:24.535034Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:20:24.535044Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:20:24.535063Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:20:24.535908Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:20:24.937398Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 402ms prompt_tokens=2 cached_tokens=0 generation_time_ms=402 +2026-10-11T05:20:24.937482Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:20:24.938431Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:20:24.938464Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:20:24.938467Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:20:24.938468Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:20:24.938470Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:20:24.938471Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:20:24.938472Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:20:24.938474Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:20:24.938475Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:20:24.938476Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:20:24.938477Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:20:24.938479Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:20:24.938480Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:20:25.410293Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:20:26.511738Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 1090ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=1090 +2026-10-11T05:20:28.824568Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2277ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2277 +2026-10-11T05:20:28.825141Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2277ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2277 +2026-10-11T05:20:28.825505Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2278ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2278 +2026-10-11T05:20:28.825767Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2278ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2278 +2026-10-11T05:20:34.303234Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5457ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5457 +2026-10-11T05:20:34.303811Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5446ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5446 +2026-10-11T05:20:34.304153Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5458ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5458 +2026-10-11T05:20:34.304452Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5458ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5458 +2026-10-11T05:20:34.304728Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5447ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5447 +2026-10-11T05:20:34.304983Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5447ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5447 +2026-10-11T05:20:34.305243Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5447ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5447 +2026-10-11T05:20:34.305532Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5447ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5447 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r2-paged-a.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r2-paged-a.log new file mode 100644 index 000000000..5f10dbfd4 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r2-paged-a.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=paged +2026-10-11T05:22:50.545955Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:22:50.545991Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:22:50.546100Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:22:50.546179Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:22:50.546181Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:22:50.546183Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:22:51.156145Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:22:51.156713Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:22:51.156896Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:22:51.777291Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:22:51.777796Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:22:52.358486Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.581s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.580619913 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:22:52.363048Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:22:52.363065Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T05:22:52.363328Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:22:52.363338Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:22:52.363357Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:22:52.364196Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:22:52.772961Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 409ms prompt_tokens=2 cached_tokens=0 generation_time_ms=409 +2026-10-11T05:22:52.773046Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:22:52.773988Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:22:52.774014Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:22:52.774017Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:22:52.774019Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:22:52.774020Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:22:52.774022Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:22:52.774023Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:22:52.774024Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:22:52.774025Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:22:52.774026Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:22:52.774028Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:22:52.774029Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:22:52.774030Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:22:53.259868Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:22:54.359234Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 1092ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=1092 +2026-10-11T05:22:56.680459Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2289ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2289 +2026-10-11T05:22:56.681094Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2290ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2290 +2026-10-11T05:22:56.681505Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2290ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2290 +2026-10-11T05:22:56.681839Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2291ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2291 +2026-10-11T05:23:02.129890Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5427ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5427 +2026-10-11T05:23:02.130776Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5428ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5428 +2026-10-11T05:23:02.131203Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5417ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5417 +2026-10-11T05:23:02.131564Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5417ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5417 +2026-10-11T05:23:02.131900Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5417ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5417 +2026-10-11T05:23:02.132216Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5418ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5418 +2026-10-11T05:23:02.132535Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5418ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5418 +2026-10-11T05:23:02.132857Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5418ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5418 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r2-paged-b.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r2-paged-b.log new file mode 100644 index 000000000..3811d2bde --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r2-paged-b.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=paged +2026-10-11T05:21:36.550594Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:21:36.550627Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:21:36.550737Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:21:36.550838Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:21:36.550841Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:21:36.550844Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:21:37.169282Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:21:37.169847Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:21:37.170029Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:21:37.775525Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:21:37.775637Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:21:38.391357Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.616s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.615706236 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:21:38.396126Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:21:38.396148Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T05:21:38.396449Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:21:38.396460Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:21:38.396481Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:21:38.397359Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:21:38.782847Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 386ms prompt_tokens=2 cached_tokens=0 generation_time_ms=386 +2026-10-11T05:21:38.782924Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:21:38.783855Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:21:38.783876Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:21:38.783879Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:21:38.783881Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:21:38.783882Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:21:38.783884Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:21:38.783885Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:21:38.783886Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:21:38.783887Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:21:38.783889Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:21:38.783890Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:21:38.783891Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:21:38.783893Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:21:39.262023Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:21:40.386395Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 1117ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=1117 +2026-10-11T05:21:42.790835Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2372ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2372 +2026-10-11T05:21:42.791420Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2373ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2373 +2026-10-11T05:21:42.791846Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2373ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2373 +2026-10-11T05:21:42.792206Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2374ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2374 +2026-10-11T05:21:48.356732Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5533ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5533 +2026-10-11T05:21:48.357579Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5533ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5533 +2026-10-11T05:21:48.358040Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5534ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5534 +2026-10-11T05:21:48.358383Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5534ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5534 +2026-10-11T05:21:48.358696Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5535ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5535 +2026-10-11T05:21:48.359003Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5535ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5535 +2026-10-11T05:21:48.359309Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5535ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5535 +2026-10-11T05:21:48.359610Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5548ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5548 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r3-dense.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r3-dense.log new file mode 100644 index 000000000..1e205d137 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r3-dense.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=dense +2026-10-11T05:26:32.111806Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:26:32.111838Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:26:32.111944Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:26:32.112002Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:26:32.112005Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:26:32.112007Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:26:32.694753Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:26:32.695260Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:26:32.695428Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:26:33.272376Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:26:33.272858Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:26:33.887875Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.615s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.614963117 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:26:33.892667Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:26:33.892689Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4) +2026-10-11T05:26:33.892985Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:26:33.892996Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:26:33.893017Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:26:33.893901Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:26:34.304007Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 410ms prompt_tokens=2 cached_tokens=0 generation_time_ms=410 +2026-10-11T05:26:34.304086Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:26:34.305014Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:26:34.305042Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:26:34.305045Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:26:34.305047Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:26:34.305048Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:26:34.305049Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:26:34.305050Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:26:34.305052Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:26:34.305053Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:26:34.305054Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:26:34.305055Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:26:34.305057Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:26:34.305058Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:26:34.857255Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:26:35.971250Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 1103ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=1103 +2026-10-11T05:26:38.337559Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2331ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2331 +2026-10-11T05:26:38.337993Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2331ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2331 +2026-10-11T05:26:38.338290Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2332ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2332 +2026-10-11T05:26:38.338552Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2332ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2332 +2026-10-11T05:26:43.762464Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5404ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5404 +2026-10-11T05:26:43.763074Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5405ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5405 +2026-10-11T05:26:43.763405Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5392ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5392 +2026-10-11T05:26:43.763686Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5392ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5392 +2026-10-11T05:26:43.763943Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5406ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5406 +2026-10-11T05:26:43.764183Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5393ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5393 +2026-10-11T05:26:43.764420Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5393ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5393 +2026-10-11T05:26:43.764672Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5393ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5393 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r3-paged-a.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r3-paged-a.log new file mode 100644 index 000000000..63270f727 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r3-paged-a.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=paged +2026-10-11T05:25:18.202568Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:25:18.202598Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:25:18.202703Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:25:18.202745Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:25:18.202747Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:25:18.202749Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:25:18.785382Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:25:18.785904Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:25:18.786073Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:25:19.361129Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:25:19.361256Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:25:19.985302Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.624s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.624017463 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:25:19.990061Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:25:19.990083Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T05:25:19.990368Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:25:19.990379Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:25:19.990399Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:25:19.991271Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:25:20.405569Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 414ms prompt_tokens=2 cached_tokens=0 generation_time_ms=414 +2026-10-11T05:25:20.405640Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:25:20.406563Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:25:20.406590Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:25:20.406594Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:25:20.406595Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:25:20.406597Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:25:20.406598Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:25:20.406599Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:25:20.406600Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:25:20.406602Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:25:20.406603Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:25:20.406604Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:25:20.406605Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:25:20.406607Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:25:20.914386Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:25:22.027095Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 1101ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=1101 +2026-10-11T05:25:24.357518Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2295ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2295 +2026-10-11T05:25:24.358159Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2295ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2295 +2026-10-11T05:25:24.358586Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2296ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2296 +2026-10-11T05:25:24.358925Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2296ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2296 +2026-10-11T05:25:29.845938Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5465ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5465 +2026-10-11T05:25:29.846673Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5453ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5453 +2026-10-11T05:25:29.847117Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5454ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5454 +2026-10-11T05:25:29.847461Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5454ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5454 +2026-10-11T05:25:29.847779Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5454ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5454 +2026-10-11T05:25:29.848090Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5454ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5454 +2026-10-11T05:25:29.848394Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5455ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5455 +2026-10-11T05:25:29.848705Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5455ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5455 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r3-paged-b.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r3-paged-b.log new file mode 100644 index 000000000..24975fec1 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r3-paged-b.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=paged +2026-10-11T05:24:04.374613Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:24:04.374643Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:24:04.374748Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:24:04.374827Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:24:04.374829Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:24:04.374831Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:24:04.956727Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:24:04.957254Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:24:04.957432Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:24:05.527322Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:24:05.527804Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:24:06.132944Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.605s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.605083711 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:24:06.137740Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:24:06.143432Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T05:24:06.143877Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:24:06.143889Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:24:06.143910Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:24:06.144788Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:24:06.558254Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 414ms prompt_tokens=2 cached_tokens=0 generation_time_ms=414 +2026-10-11T05:24:06.558298Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:24:06.559205Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:24:06.559233Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:24:06.559236Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:24:06.559237Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:24:06.559239Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:24:06.559240Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:24:06.559241Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:24:06.559242Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:24:06.559244Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:24:06.559245Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:24:06.559246Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:24:06.559247Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:24:06.559249Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:24:07.113075Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:24:08.219567Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 1099ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=1099 +2026-10-11T05:24:10.547385Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2295ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2295 +2026-10-11T05:24:10.547882Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2295ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2295 +2026-10-11T05:24:10.548231Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2295ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2295 +2026-10-11T05:24:10.548527Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2296ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2296 +2026-10-11T05:24:16.027890Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5458ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5458 +2026-10-11T05:24:16.028591Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5458ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5458 +2026-10-11T05:24:16.029017Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5446ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5446 +2026-10-11T05:24:16.029367Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5459ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5459 +2026-10-11T05:24:16.029680Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5459ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5459 +2026-10-11T05:24:16.029993Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5459ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5459 +2026-10-11T05:24:16.030297Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5460ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5460 +2026-10-11T05:24:16.030619Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5460ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5460 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r4-dense.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r4-dense.log new file mode 100644 index 000000000..9165cfed1 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r4-dense.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=dense +2026-10-11T05:29:00.019095Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:29:00.019125Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:29:00.019230Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:29:00.019291Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:29:00.019293Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:29:00.019296Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:29:00.602090Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:29:00.602613Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:29:00.602781Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:29:01.176021Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:29:01.176037Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:29:01.775982Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.600s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.599925807 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:29:01.780465Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:29:01.780485Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4) +2026-10-11T05:29:01.780749Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:29:01.780760Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:29:01.780779Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:29:01.781652Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:29:02.172534Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 391ms prompt_tokens=2 cached_tokens=0 generation_time_ms=391 +2026-10-11T05:29:02.172571Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:29:02.173445Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:29:02.173460Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:29:02.173463Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:29:02.173465Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:29:02.173466Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:29:02.173468Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:29:02.173469Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:29:02.173470Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:29:02.173471Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:29:02.173472Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:29:02.173474Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:29:02.173475Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:29:02.173476Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:29:02.732916Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:29:03.844592Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 1104ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=1104 +2026-10-11T05:29:06.204866Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2328ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2328 +2026-10-11T05:29:06.205280Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2328ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2328 +2026-10-11T05:29:06.205587Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2329ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2329 +2026-10-11T05:29:06.205833Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2329ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2329 +2026-10-11T05:29:11.753610Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5514ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5514 +2026-10-11T05:29:11.754214Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5514ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5514 +2026-10-11T05:29:11.754573Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5515ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5515 +2026-10-11T05:29:11.754882Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5527ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5527 +2026-10-11T05:29:11.755155Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5515ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5515 +2026-10-11T05:29:11.755412Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5516ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5516 +2026-10-11T05:29:11.755670Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5516ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5516 +2026-10-11T05:29:11.755944Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5528ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5528 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r4-paged-a.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r4-paged-a.log new file mode 100644 index 000000000..3cfd04f4f --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r4-paged-a.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=paged +2026-10-11T05:27:45.959932Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:27:45.959964Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:27:45.960073Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:27:45.960159Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:27:45.960161Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:27:45.960164Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:27:46.578342Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:27:46.578937Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:27:46.579115Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:27:47.193699Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:27:47.193707Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:27:47.773217Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.579s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.579479455 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:27:47.777670Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:27:47.777686Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T05:27:47.777947Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:27:47.777957Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:27:47.777976Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:27:47.778823Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:27:48.176442Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 398ms prompt_tokens=2 cached_tokens=0 generation_time_ms=398 +2026-10-11T05:27:48.176484Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:27:48.177436Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:27:48.177468Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:27:48.177471Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:27:48.177473Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:27:48.177475Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:27:48.177476Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:27:48.177477Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:27:48.177478Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:27:48.177480Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:27:48.177481Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:27:48.177482Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:27:48.177483Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:27:48.177485Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:27:48.702838Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:27:49.812322Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 1102ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=1102 +2026-10-11T05:27:52.159300Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2316ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2316 +2026-10-11T05:27:52.159855Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2317ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2317 +2026-10-11T05:27:52.160253Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2317ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2317 +2026-10-11T05:27:52.160596Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2317ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2317 +2026-10-11T05:27:57.733634Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5551ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5551 +2026-10-11T05:27:57.734519Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5552ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5552 +2026-10-11T05:27:57.735190Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5540ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5540 +2026-10-11T05:27:57.735818Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5540ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5540 +2026-10-11T05:27:57.736423Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5541ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5541 +2026-10-11T05:27:57.737031Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5542ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5542 +2026-10-11T05:27:57.737625Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5542ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5542 +2026-10-11T05:27:57.738225Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5556ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5556 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r4-paged-b.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r4-paged-b.log new file mode 100644 index 000000000..b295c1b23 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r4-paged-b.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=paged +2026-10-11T05:30:13.960889Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:30:13.960920Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:30:13.961028Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:30:13.961095Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:30:13.961097Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:30:13.961099Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:30:14.571177Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:30:14.571717Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:30:14.571895Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:30:15.184833Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:30:15.185326Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:30:15.775015Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.590s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.589665645 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:30:15.779489Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:30:15.779503Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T05:30:15.779763Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:30:15.779774Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:30:15.779794Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:30:15.780648Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:30:16.162099Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 382ms prompt_tokens=2 cached_tokens=0 generation_time_ms=382 +2026-10-11T05:30:16.162180Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:30:16.163108Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:30:16.163138Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:30:16.163142Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:30:16.163143Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:30:16.163145Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:30:16.163146Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:30:16.163147Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:30:16.163149Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:30:16.163150Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:30:16.163151Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:30:16.163152Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:30:16.163153Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:30:16.163155Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:30:16.694880Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:30:17.789054Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 1087ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=1087 +2026-10-11T05:30:20.124645Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2304ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2304 +2026-10-11T05:30:20.125126Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2304ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2304 +2026-10-11T05:30:20.125497Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2305ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2305 +2026-10-11T05:30:20.125828Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2305ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2305 +2026-10-11T05:30:25.592231Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5433ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5433 +2026-10-11T05:30:25.593197Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5446ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5446 +2026-10-11T05:30:25.593900Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5447ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5447 +2026-10-11T05:30:25.594592Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5435ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5435 +2026-10-11T05:30:25.595238Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5436ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5436 +2026-10-11T05:30:25.595898Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5449ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5449 +2026-10-11T05:30:25.596548Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5437ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5437 +2026-10-11T05:30:25.597198Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5438ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5438 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r5-dense.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r5-dense.log new file mode 100644 index 000000000..64c61ebab --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r5-dense.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=dense +2026-10-11T05:31:27.802520Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:31:27.802563Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:31:27.802672Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:31:27.802748Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:31:27.802751Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:31:27.802753Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:31:28.420167Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:31:28.420734Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:31:28.420914Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:31:29.046839Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:31:29.047633Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:31:29.634532Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.587s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.586847239 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:31:29.639016Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:31:29.639030Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4) +2026-10-11T05:31:29.639276Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:31:29.639286Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:31:29.639305Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:31:29.640153Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:31:30.029938Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 390ms prompt_tokens=2 cached_tokens=0 generation_time_ms=390 +2026-10-11T05:31:30.029982Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:31:30.030907Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:31:30.030934Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:31:30.030937Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:31:30.030939Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:31:30.030940Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:31:30.030941Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:31:30.030943Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:31:30.030944Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:31:30.030945Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:31:30.030947Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:31:30.030948Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:31:30.030949Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:31:30.030950Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:31:30.549806Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:31:31.654838Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 1098ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=1098 +2026-10-11T05:31:33.983852Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2293ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2293 +2026-10-11T05:31:33.984309Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2293ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2293 +2026-10-11T05:31:33.984606Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2294ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2294 +2026-10-11T05:31:33.984851Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2294ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2294 +2026-10-11T05:31:39.492518Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5486ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5486 +2026-10-11T05:31:39.493249Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5475ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5475 +2026-10-11T05:31:39.493585Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5475ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5475 +2026-10-11T05:31:39.493885Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5475ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5475 +2026-10-11T05:31:39.494151Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5488ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5488 +2026-10-11T05:31:39.494396Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5476ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5476 +2026-10-11T05:31:39.494651Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5476ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5476 +2026-10-11T05:31:39.494911Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5476ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5476 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r5-paged-a.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r5-paged-a.log new file mode 100644 index 000000000..aa05b20e7 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r5-paged-a.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=paged +2026-10-11T05:33:56.017634Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:33:56.017666Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:33:56.017775Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:33:56.017871Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:33:56.017873Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:33:56.017875Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:33:56.629459Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:33:56.630024Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:33:56.630203Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:33:57.241442Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:33:57.241920Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:33:57.827786Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.586s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.585801553 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:33:57.832245Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:33:57.832258Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T05:33:57.832517Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:33:57.832531Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:33:57.832552Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:33:57.833392Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:33:58.292842Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 460ms prompt_tokens=2 cached_tokens=0 generation_time_ms=460 +2026-10-11T05:33:58.292924Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:33:58.293841Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:33:58.293866Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:33:58.293869Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:33:58.293871Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:33:58.293872Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:33:58.293874Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:33:58.293875Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:33:58.293876Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:33:58.293878Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:33:58.293879Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:33:58.293881Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:33:58.293882Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:33:58.293883Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:33:58.757417Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:33:59.868769Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 1104ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=1104 +2026-10-11T05:34:02.199421Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2299ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2299 +2026-10-11T05:34:02.200140Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2300ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2300 +2026-10-11T05:34:02.200569Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2300ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2300 +2026-10-11T05:34:02.200904Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2301ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2301 +2026-10-11T05:34:07.818309Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5594ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5594 +2026-10-11T05:34:07.819484Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5582ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5582 +2026-10-11T05:34:07.820268Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5596ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5596 +2026-10-11T05:34:07.820978Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5596ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5596 +2026-10-11T05:34:07.821666Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5597ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5597 +2026-10-11T05:34:07.822331Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5585ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5585 +2026-10-11T05:34:07.822989Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5599ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5599 +2026-10-11T05:34:07.823668Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5587ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5587 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r5-paged-b.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r5-paged-b.log new file mode 100644 index 000000000..39319f4b1 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p4096-r5-paged-b.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=paged +2026-10-11T05:32:41.736331Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:32:41.736363Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:32:41.736468Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:32:41.736545Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:32:41.736548Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:32:41.736551Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:32:42.322093Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:32:42.322599Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:32:42.322772Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:32:42.903431Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:32:42.903525Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:32:43.488756Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.585s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.58518246 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:32:43.493208Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:32:43.493221Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T05:32:43.493475Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:32:43.493486Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:32:43.493505Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:32:43.494352Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:32:43.870341Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 376ms prompt_tokens=2 cached_tokens=0 generation_time_ms=376 +2026-10-11T05:32:43.870379Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:32:43.871247Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:32:43.871265Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:32:43.871268Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:32:43.871270Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:32:43.871271Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:32:43.871272Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:32:43.871274Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:32:43.871275Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:32:43.871276Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:32:43.871277Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:32:43.871278Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:32:43.871280Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:32:43.871281Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:32:44.484170Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:32:45.606544Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 1111ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=1111 +2026-10-11T05:32:48.001837Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2359ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2359 +2026-10-11T05:32:48.002313Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2360ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2360 +2026-10-11T05:32:48.002698Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2360ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2360 +2026-10-11T05:32:48.003019Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3465 prompt tokens, total 2360ms prompt_tokens=3465 cached_tokens=0 generation_time_ms=2360 +2026-10-11T05:32:53.800773Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5767ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5767 +2026-10-11T05:32:53.801580Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5781ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5781 +2026-10-11T05:32:53.801998Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5768ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5768 +2026-10-11T05:32:53.802351Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5782ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5782 +2026-10-11T05:32:53.802698Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5782ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5782 +2026-10-11T05:32:53.803028Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5783ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5783 +2026-10-11T05:32:53.803350Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5770ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5770 +2026-10-11T05:32:53.803690Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3462/3465 prompt tokens, total 5770ms prompt_tokens=3465 cached_tokens=3462 generation_time_ms=5770 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r1-dense.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r1-dense.log new file mode 100644 index 000000000..2a09ba7a9 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r1-dense.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=dense +2026-10-11T04:59:46.785751Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T04:59:46.785782Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T04:59:46.785888Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T04:59:46.785949Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T04:59:46.785951Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T04:59:46.785953Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T04:59:47.365325Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T04:59:47.365859Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T04:59:47.366029Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T04:59:47.945671Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T04:59:47.945780Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T04:59:48.568264Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.622s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.622457414 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T04:59:48.573197Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T04:59:48.573220Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4) +2026-10-11T04:59:48.573537Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T04:59:48.573549Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T04:59:48.573571Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T04:59:48.574462Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T04:59:48.976171Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 402ms prompt_tokens=2 cached_tokens=0 generation_time_ms=402 +2026-10-11T04:59:48.976212Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T04:59:48.977103Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T04:59:48.977129Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T04:59:48.977133Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T04:59:48.977134Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T04:59:48.977135Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T04:59:48.977137Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T04:59:48.977138Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T04:59:48.977139Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T04:59:48.977140Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T04:59:48.977142Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T04:59:48.977143Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T04:59:48.977144Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T04:59:48.977145Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T04:59:49.536308Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T04:59:50.363442Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 825ms prompt_tokens=441 cached_tokens=0 generation_time_ms=825 +2026-10-11T04:59:51.747013Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1358ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1358 +2026-10-11T04:59:51.747343Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1359ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1359 +2026-10-11T04:59:51.747644Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1359ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1359 +2026-10-11T04:59:51.747851Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1359ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1359 +2026-10-11T04:59:56.840880Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5052ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5052 +2026-10-11T04:59:56.841543Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5085ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5085 +2026-10-11T04:59:56.842052Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5054ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5054 +2026-10-11T04:59:56.842499Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5086ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5086 +2026-10-11T04:59:56.842930Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5054ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5054 +2026-10-11T04:59:56.843352Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5055ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5055 +2026-10-11T04:59:56.843773Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5087ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5087 +2026-10-11T04:59:56.844168Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5087ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5087 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r1-paged-a.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r1-paged-a.log new file mode 100644 index 000000000..b3f3f7c93 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r1-paged-a.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=paged +2026-10-11T04:58:33.379171Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T04:58:33.379242Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T04:58:33.379733Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T04:58:33.379861Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T04:58:33.379864Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T04:58:33.379866Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T04:58:34.016507Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T04:58:34.027444Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T04:58:34.027777Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T04:58:34.648403Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T04:58:34.648412Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T04:58:35.221989Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.574s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.57354536 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T04:58:35.226483Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T04:58:35.226532Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T04:58:35.226838Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T04:58:35.226848Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T04:58:35.226959Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T04:58:35.229186Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T04:58:36.427915Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 1200ms prompt_tokens=2 cached_tokens=0 generation_time_ms=1200 +2026-10-11T04:58:36.427968Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T04:58:36.430044Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T04:58:36.430069Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T04:58:36.430073Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T04:58:36.430074Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T04:58:36.430076Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T04:58:36.430077Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T04:58:36.430078Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T04:58:36.430080Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T04:58:36.430081Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T04:58:36.430082Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T04:58:36.430084Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T04:58:36.430085Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T04:58:36.430086Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T04:58:37.106153Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T04:58:37.963894Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 855ms prompt_tokens=441 cached_tokens=0 generation_time_ms=855 +2026-10-11T04:58:39.382898Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1392ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1392 +2026-10-11T04:58:39.383540Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1392ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1392 +2026-10-11T04:58:39.383867Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1393ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1393 +2026-10-11T04:58:39.384132Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1393ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1393 +2026-10-11T04:58:44.537235Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5145ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5145 +2026-10-11T04:58:44.538095Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5146ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5146 +2026-10-11T04:58:44.538578Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5115ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5115 +2026-10-11T04:58:44.538898Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5147ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5147 +2026-10-11T04:58:44.539210Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5116ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5116 +2026-10-11T04:58:44.539483Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5116ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5116 +2026-10-11T04:58:44.539746Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5117ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5117 +2026-10-11T04:58:44.539989Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5148ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5148 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r1-paged-b.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r1-paged-b.log new file mode 100644 index 000000000..9827b2835 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r1-paged-b.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=paged +2026-10-11T05:00:58.992651Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:00:58.992685Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:00:58.992793Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:00:58.992870Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:00:58.992873Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:00:58.992875Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:00:59.619326Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:00:59.619932Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:00:59.620116Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:01:00.247125Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:01:00.247825Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:01:00.840160Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.592s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.592283624 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:01:00.844642Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:01:00.844654Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T05:01:00.844914Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:01:00.844924Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:01:00.844942Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:01:00.845788Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:01:01.249786Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 404ms prompt_tokens=2 cached_tokens=0 generation_time_ms=404 +2026-10-11T05:01:01.249829Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:01:01.250749Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:01:01.250774Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:01:01.250778Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:01:01.250780Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:01:01.250781Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:01:01.250783Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:01:01.250784Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:01:01.250786Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:01:01.250787Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:01:01.250788Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:01:01.250790Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:01:01.250791Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:01:01.250793Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:01:01.739192Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:01:02.564669Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 824ms prompt_tokens=441 cached_tokens=0 generation_time_ms=824 +2026-10-11T05:01:03.935083Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1345ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1345 +2026-10-11T05:01:03.935489Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1345ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1345 +2026-10-11T05:01:03.935811Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1345ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1345 +2026-10-11T05:01:03.936053Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1346ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1346 +2026-10-11T05:01:09.122347Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5178ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5178 +2026-10-11T05:01:09.123351Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5149ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5149 +2026-10-11T05:01:09.123980Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5179ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5179 +2026-10-11T05:01:09.124525Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5180ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5180 +2026-10-11T05:01:09.125065Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5181ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5181 +2026-10-11T05:01:09.125585Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5181ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5181 +2026-10-11T05:01:09.126091Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5181ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5181 +2026-10-11T05:01:09.126613Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5182ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5182 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r2-dense.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r2-dense.log new file mode 100644 index 000000000..e7d69d1d3 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r2-dense.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=dense +2026-10-11T05:02:11.302249Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:02:11.302281Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:02:11.302388Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:02:11.302469Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:02:11.302471Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:02:11.302474Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:02:11.916774Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:02:11.917349Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:02:11.917526Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:02:12.528014Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:02:12.528026Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:02:13.146813Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.619s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.618759715 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:02:13.151565Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:02:13.151589Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4) +2026-10-11T05:02:13.151878Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:02:13.151889Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:02:13.151909Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:02:13.152787Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:02:13.599274Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 447ms prompt_tokens=2 cached_tokens=0 generation_time_ms=447 +2026-10-11T05:02:13.599315Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:02:13.600209Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:02:13.600229Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:02:13.600232Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:02:13.600233Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:02:13.600235Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:02:13.600236Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:02:13.600237Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:02:13.600238Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:02:13.600240Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:02:13.600241Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:02:13.600242Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:02:13.600243Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:02:13.600245Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:02:14.022630Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:02:14.864285Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 840ms prompt_tokens=441 cached_tokens=0 generation_time_ms=840 +2026-10-11T05:02:16.271947Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1383ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1383 +2026-10-11T05:02:16.272330Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1383ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1383 +2026-10-11T05:02:16.272607Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1383ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1383 +2026-10-11T05:02:16.272844Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1384ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1384 +2026-10-11T05:02:21.467041Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5185ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5185 +2026-10-11T05:02:21.467871Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5156ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5156 +2026-10-11T05:02:21.468187Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5156ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5156 +2026-10-11T05:02:21.468469Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5186ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5186 +2026-10-11T05:02:21.468729Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5186ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5186 +2026-10-11T05:02:21.468959Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5186ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5186 +2026-10-11T05:02:21.469178Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5187ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5187 +2026-10-11T05:02:21.469380Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5187ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5187 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r2-paged-a.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r2-paged-a.log new file mode 100644 index 000000000..3e38b18a2 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r2-paged-a.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=paged +2026-10-11T05:04:36.046080Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:04:36.046110Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:04:36.046213Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:04:36.046286Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:04:36.046288Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:04:36.046290Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:04:36.618899Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:04:36.619399Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:04:36.619575Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:04:37.192036Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:04:37.192122Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:04:37.783957Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.592s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.591822058 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:04:37.788442Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:04:37.788454Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T05:04:37.788705Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:04:37.788716Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:04:37.788737Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:04:37.789575Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:04:38.193885Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 404ms prompt_tokens=2 cached_tokens=0 generation_time_ms=404 +2026-10-11T05:04:38.193924Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:04:38.194810Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:04:38.194828Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:04:38.194831Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:04:38.194833Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:04:38.194834Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:04:38.194836Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:04:38.194837Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:04:38.194838Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:04:38.194839Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:04:38.194840Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:04:38.194842Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:04:38.194843Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:04:38.194844Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:04:38.782982Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:04:39.601075Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 816ms prompt_tokens=441 cached_tokens=0 generation_time_ms=816 +2026-10-11T05:04:41.017784Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1392ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1392 +2026-10-11T05:04:41.018164Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1393ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1393 +2026-10-11T05:04:41.018441Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1393ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1393 +2026-10-11T05:04:41.018701Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1393ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1393 +2026-10-11T05:04:46.229104Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5201ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5201 +2026-10-11T05:04:46.229955Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5171ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5171 +2026-10-11T05:04:46.230605Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5203ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5203 +2026-10-11T05:04:46.231193Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5203ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5203 +2026-10-11T05:04:46.231765Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5204ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5204 +2026-10-11T05:04:46.232298Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5204ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5204 +2026-10-11T05:04:46.232830Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5205ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5205 +2026-10-11T05:04:46.233348Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5205ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5205 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r2-paged-b.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r2-paged-b.log new file mode 100644 index 000000000..f04f40b1e --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r2-paged-b.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=paged +2026-10-11T05:03:23.688671Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:03:23.688702Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:03:23.688811Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:03:23.688876Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:03:23.688879Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:03:23.688881Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:03:24.265718Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:03:24.266227Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:03:24.266401Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:03:24.840885Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:03:24.841391Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:03:25.451313Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.610s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.609890023 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:03:25.456081Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:03:25.456103Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T05:03:25.456388Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:03:25.456398Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:03:25.456419Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:03:25.457290Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:03:25.867340Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 410ms prompt_tokens=2 cached_tokens=0 generation_time_ms=410 +2026-10-11T05:03:25.867381Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:03:25.868296Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:03:25.868319Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:03:25.868323Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:03:25.868325Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:03:25.868326Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:03:25.868327Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:03:25.868329Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:03:25.868330Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:03:25.868332Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:03:25.868333Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:03:25.868334Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:03:25.868336Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:03:25.868337Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:03:26.411424Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:03:27.249130Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 835ms prompt_tokens=441 cached_tokens=0 generation_time_ms=835 +2026-10-11T05:03:28.667840Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1391ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1391 +2026-10-11T05:03:28.668271Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1392ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1392 +2026-10-11T05:03:28.668620Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1392ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1392 +2026-10-11T05:03:28.668889Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1393ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1393 +2026-10-11T05:03:33.819179Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5142ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5142 +2026-10-11T05:03:33.819772Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5142ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5142 +2026-10-11T05:03:33.820130Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5142ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5142 +2026-10-11T05:03:33.820432Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5143ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5143 +2026-10-11T05:03:33.820715Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5143ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5143 +2026-10-11T05:03:33.820981Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5143ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5143 +2026-10-11T05:03:33.821238Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5112ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5112 +2026-10-11T05:03:33.821487Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5112ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5112 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r3-dense.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r3-dense.log new file mode 100644 index 000000000..f7bd2935c --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r3-dense.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=dense +2026-10-11T05:08:13.625839Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:08:13.625869Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:08:13.625976Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:08:13.626039Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:08:13.626041Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:08:13.626044Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:08:14.215282Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:08:14.215781Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:08:14.215957Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:08:14.788063Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:08:14.788156Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:08:15.357763Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.570s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.569586981 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:08:15.362339Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:08:15.362354Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4) +2026-10-11T05:08:15.362608Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:08:15.362619Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:08:15.362639Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:08:15.363491Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:08:15.758070Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 395ms prompt_tokens=2 cached_tokens=0 generation_time_ms=395 +2026-10-11T05:08:15.758145Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:08:15.759042Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:08:15.759063Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:08:15.759066Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:08:15.759068Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:08:15.759069Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:08:15.759070Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:08:15.759072Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:08:15.759073Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:08:15.759074Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:08:15.759075Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:08:15.759076Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:08:15.759078Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:08:15.759079Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:08:16.354540Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:08:17.189380Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 833ms prompt_tokens=441 cached_tokens=0 generation_time_ms=833 +2026-10-11T05:08:18.625441Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1412ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1412 +2026-10-11T05:08:18.625855Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1412ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1412 +2026-10-11T05:08:18.626145Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1412ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1412 +2026-10-11T05:08:18.626374Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1413ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1413 +2026-10-11T05:08:23.778176Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5143ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5143 +2026-10-11T05:08:23.779035Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5144ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5144 +2026-10-11T05:08:23.779500Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5144ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5144 +2026-10-11T05:08:23.779797Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5144ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5144 +2026-10-11T05:08:23.780052Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5145ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5145 +2026-10-11T05:08:23.780295Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5145ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5145 +2026-10-11T05:08:23.780521Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5145ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5145 +2026-10-11T05:08:23.780728Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5145ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5145 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r3-paged-a.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r3-paged-a.log new file mode 100644 index 000000000..54104e3dc --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r3-paged-a.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=paged +2026-10-11T05:07:01.004226Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:07:01.004261Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:07:01.004367Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:07:01.004422Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:07:01.004425Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:07:01.004427Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:07:01.616299Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:07:01.616870Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:07:01.617050Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:07:02.229384Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:07:02.229424Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:07:02.821615Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.592s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.592164926 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:07:02.826078Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:07:02.826092Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T05:07:02.826348Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:07:02.826358Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:07:02.826376Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:07:02.827237Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:07:03.226215Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 399ms prompt_tokens=2 cached_tokens=0 generation_time_ms=399 +2026-10-11T05:07:03.226260Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:07:03.227181Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:07:03.227209Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:07:03.227213Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:07:03.227214Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:07:03.227215Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:07:03.227217Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:07:03.227218Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:07:03.227219Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:07:03.227220Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:07:03.227221Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:07:03.227223Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:07:03.227224Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:07:03.227225Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:07:03.722934Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:07:04.545348Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 821ms prompt_tokens=441 cached_tokens=0 generation_time_ms=821 +2026-10-11T05:07:05.950803Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1380ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1380 +2026-10-11T05:07:05.951209Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1381ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1381 +2026-10-11T05:07:05.951535Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1381ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1381 +2026-10-11T05:07:05.951783Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1381ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1381 +2026-10-11T05:07:11.436105Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5475ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5475 +2026-10-11T05:07:11.436721Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5444ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5444 +2026-10-11T05:07:11.437100Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5445ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5445 +2026-10-11T05:07:11.437417Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5477ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5477 +2026-10-11T05:07:11.437720Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5445ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5445 +2026-10-11T05:07:11.437992Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5445ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5445 +2026-10-11T05:07:11.438256Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5477ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5477 +2026-10-11T05:07:11.438521Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5478ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5478 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r3-paged-b.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r3-paged-b.log new file mode 100644 index 000000000..7ee1d8aa5 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r3-paged-b.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=paged +2026-10-11T05:05:48.537994Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:05:48.538025Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:05:48.538132Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:05:48.538171Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:05:48.538173Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:05:48.538175Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:05:49.120457Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:05:49.120982Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:05:49.121158Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:05:49.695640Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:05:49.695639Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:05:50.310869Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.615s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.615193164 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:05:50.315658Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:05:50.315679Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T05:05:50.315964Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:05:50.315975Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:05:50.315995Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:05:50.316879Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:05:50.690237Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 374ms prompt_tokens=2 cached_tokens=0 generation_time_ms=374 +2026-10-11T05:05:50.690281Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:05:50.691170Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:05:50.691197Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:05:50.691200Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:05:50.691201Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:05:50.691203Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:05:50.691204Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:05:50.691205Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:05:50.691206Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:05:50.691208Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:05:50.691209Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:05:50.691210Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:05:50.691212Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:05:50.691213Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:05:51.266814Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:05:52.094520Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 825ms prompt_tokens=441 cached_tokens=0 generation_time_ms=825 +2026-10-11T05:05:53.520712Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1400ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1400 +2026-10-11T05:05:53.521195Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1400ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1400 +2026-10-11T05:05:53.521482Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1401ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1401 +2026-10-11T05:05:53.521721Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1401ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1401 +2026-10-11T05:05:58.801332Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5239ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5239 +2026-10-11T05:05:58.801959Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5271ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5271 +2026-10-11T05:05:58.802327Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5240ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5240 +2026-10-11T05:05:58.802651Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5240ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5240 +2026-10-11T05:05:58.802941Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5240ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5240 +2026-10-11T05:05:58.803235Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5272ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5272 +2026-10-11T05:05:58.803499Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5273ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5273 +2026-10-11T05:05:58.803763Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5273ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5273 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r4-dense.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r4-dense.log new file mode 100644 index 000000000..875f004ef --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r4-dense.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=dense +2026-10-11T05:10:38.444002Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:10:38.444035Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:10:38.444142Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:10:38.444234Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:10:38.444236Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:10:38.444238Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:10:39.027439Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:10:39.027936Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:10:39.028107Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:10:39.604917Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:10:39.605226Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:10:40.240023Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.635s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.634750974 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:10:40.244767Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:10:40.244793Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4) +2026-10-11T05:10:40.245097Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:10:40.245108Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:10:40.245130Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:10:40.246002Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:10:40.638282Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 392ms prompt_tokens=2 cached_tokens=0 generation_time_ms=392 +2026-10-11T05:10:40.638363Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:10:40.639273Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:10:40.639299Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:10:40.639302Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:10:40.639304Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:10:40.639305Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:10:40.639307Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:10:40.639308Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:10:40.639309Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:10:40.639310Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:10:40.639312Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:10:40.639313Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:10:40.639314Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:10:40.639316Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:10:41.191512Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:10:42.013208Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 819ms prompt_tokens=441 cached_tokens=0 generation_time_ms=819 +2026-10-11T05:10:43.430806Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1392ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1392 +2026-10-11T05:10:43.431175Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1392ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1392 +2026-10-11T05:10:43.431446Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1392ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1392 +2026-10-11T05:10:43.431674Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1393ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1393 +2026-10-11T05:10:48.621519Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5181ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5181 +2026-10-11T05:10:48.622134Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5182ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5182 +2026-10-11T05:10:48.622465Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5182ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5182 +2026-10-11T05:10:48.622753Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5150ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5150 +2026-10-11T05:10:48.623015Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5182ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5182 +2026-10-11T05:10:48.623250Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5183ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5183 +2026-10-11T05:10:48.623469Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5150ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5150 +2026-10-11T05:10:48.623674Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5150ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5150 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r4-paged-a.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r4-paged-a.log new file mode 100644 index 000000000..ba7c6f721 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r4-paged-a.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=paged +2026-10-11T05:09:25.988639Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:09:25.988670Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:09:25.988778Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:09:25.988873Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:09:25.988876Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:09:25.988878Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:09:26.611721Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:09:26.612288Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:09:26.612463Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:09:27.229276Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:09:27.229347Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:09:27.843014Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.614s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.613653919 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:09:27.847843Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:09:27.847863Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T05:09:27.848150Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:09:27.848161Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:09:27.848181Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:09:27.849069Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:09:28.249717Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 401ms prompt_tokens=2 cached_tokens=0 generation_time_ms=401 +2026-10-11T05:09:28.249796Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:09:28.250699Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:09:28.250721Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:09:28.250724Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:09:28.250726Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:09:28.250727Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:09:28.250728Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:09:28.250730Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:09:28.250731Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:09:28.250732Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:09:28.250734Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:09:28.250735Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:09:28.250736Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:09:28.250738Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:09:28.710870Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:09:29.537666Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 825ms prompt_tokens=441 cached_tokens=0 generation_time_ms=825 +2026-10-11T05:09:30.972068Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1409ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1409 +2026-10-11T05:09:30.972507Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1409ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1409 +2026-10-11T05:09:30.972850Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1410ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1410 +2026-10-11T05:09:30.973104Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1410ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1410 +2026-10-11T05:09:36.223591Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5210ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5210 +2026-10-11T05:09:36.224251Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5242ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5242 +2026-10-11T05:09:36.224617Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5243ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5243 +2026-10-11T05:09:36.224920Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5243ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5243 +2026-10-11T05:09:36.225199Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5243ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5243 +2026-10-11T05:09:36.225455Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5212ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5212 +2026-10-11T05:09:36.225713Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5212ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5212 +2026-10-11T05:09:36.225971Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5213ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5213 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r4-paged-b.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r4-paged-b.log new file mode 100644 index 000000000..90930346c --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r4-paged-b.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=paged +2026-10-11T05:11:50.822481Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:11:50.822513Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:11:50.822631Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:11:50.822704Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:11:50.822706Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:11:50.822708Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:11:51.447924Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:11:51.448501Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:11:51.448695Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:11:52.075915Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:11:52.075921Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:11:52.655428Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.579s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.579471038 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:11:52.659893Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:11:52.659908Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T05:11:52.660162Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:11:52.660172Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:11:52.660190Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:11:52.661043Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:11:53.031789Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 371ms prompt_tokens=2 cached_tokens=0 generation_time_ms=371 +2026-10-11T05:11:53.031833Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:11:53.032761Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:11:53.032788Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:11:53.032792Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:11:53.032794Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:11:53.032795Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:11:53.032797Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:11:53.032798Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:11:53.032799Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:11:53.032800Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:11:53.032802Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:11:53.032803Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:11:53.032804Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:11:53.032805Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:11:53.545040Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:11:54.366390Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 820ms prompt_tokens=441 cached_tokens=0 generation_time_ms=820 +2026-10-11T05:11:55.819195Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1428ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1428 +2026-10-11T05:11:55.819620Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1428ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1428 +2026-10-11T05:11:55.819924Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1428ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1428 +2026-10-11T05:11:55.820180Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1429ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1429 +2026-10-11T05:12:01.084786Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5256ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5256 +2026-10-11T05:12:01.085460Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5257ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5257 +2026-10-11T05:12:01.085847Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5226ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5226 +2026-10-11T05:12:01.086162Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5258ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5258 +2026-10-11T05:12:01.086456Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5226ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5226 +2026-10-11T05:12:01.086751Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5227ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5227 +2026-10-11T05:12:01.087006Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5227ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5227 +2026-10-11T05:12:01.087247Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5259ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5259 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r5-dense.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r5-dense.log new file mode 100644 index 000000000..f82a17444 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r5-dense.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=dense +2026-10-11T05:13:03.233848Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:13:03.233880Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:13:03.233987Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:13:03.234049Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:13:03.234051Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:13:03.234053Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:13:03.863815Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:13:03.864390Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:13:03.864578Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:13:04.488882Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:13:04.489813Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:13:05.057509Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.568s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.567661541 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:13:05.061993Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:13:05.062007Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4) +2026-10-11T05:13:05.062284Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:13:05.062295Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:13:05.062312Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:13:05.063143Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:13:05.472151Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 409ms prompt_tokens=2 cached_tokens=0 generation_time_ms=409 +2026-10-11T05:13:05.472190Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:13:05.473104Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:13:05.473130Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:13:05.473134Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:13:05.473135Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:13:05.473137Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:13:05.473138Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:13:05.473139Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:13:05.473140Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:13:05.473142Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:13:05.473143Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:13:05.473144Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:13:05.473145Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:13:05.473147Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:13:05.965172Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:13:06.782174Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 815ms prompt_tokens=441 cached_tokens=0 generation_time_ms=815 +2026-10-11T05:13:08.188937Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1381ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1381 +2026-10-11T05:13:08.189294Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1382ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1382 +2026-10-11T05:13:08.189547Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1382ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1382 +2026-10-11T05:13:08.189751Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1382ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1382 +2026-10-11T05:13:13.457626Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5259ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5259 +2026-10-11T05:13:13.458381Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5260ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5260 +2026-10-11T05:13:13.458836Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5260ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5260 +2026-10-11T05:13:13.459248Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5261ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5261 +2026-10-11T05:13:13.459668Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5229ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5229 +2026-10-11T05:13:13.460050Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5261ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5261 +2026-10-11T05:13:13.460423Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5262ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5262 +2026-10-11T05:13:13.460799Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5262ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5262 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r5-paged-a.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r5-paged-a.log new file mode 100644 index 000000000..cd47381aa --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r5-paged-a.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=paged +2026-10-11T05:15:28.115291Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:15:28.115324Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:15:28.115430Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:15:28.115500Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:15:28.115502Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:15:28.115505Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:15:28.730131Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:15:28.730684Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:15:28.730863Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:15:29.353506Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:15:29.353992Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:15:29.933160Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.579s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.579106148 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:15:29.937644Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:15:29.937658Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T05:15:29.937907Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:15:29.937916Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:15:29.937934Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:15:29.938766Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:15:30.353065Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 414ms prompt_tokens=2 cached_tokens=0 generation_time_ms=414 +2026-10-11T05:15:30.353110Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:15:30.354016Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:15:30.354043Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:15:30.354047Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:15:30.354048Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:15:30.354050Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:15:30.354051Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:15:30.354052Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:15:30.354053Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:15:30.354055Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:15:30.354056Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:15:30.354057Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:15:30.354058Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:15:30.354060Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:15:30.872245Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:15:31.702227Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 827ms prompt_tokens=441 cached_tokens=0 generation_time_ms=827 +2026-10-11T05:15:33.134999Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1407ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1407 +2026-10-11T05:15:33.135457Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1407ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1407 +2026-10-11T05:15:33.135792Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1407ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1407 +2026-10-11T05:15:33.136355Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1408ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1408 +2026-10-11T05:15:38.418700Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5273ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5273 +2026-10-11T05:15:38.419456Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5274ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5274 +2026-10-11T05:15:38.419839Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5244ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5244 +2026-10-11T05:15:38.420169Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5275ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5275 +2026-10-11T05:15:38.420461Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5275ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5275 +2026-10-11T05:15:38.420853Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5245ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5245 +2026-10-11T05:15:38.421305Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5276ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5276 +2026-10-11T05:15:38.421610Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5245ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5245 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r5-paged-b.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r5-paged-b.log new file mode 100644 index 000000000..7f3cfd786 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-gemma3-p512-r5-paged-b.log @@ -0,0 +1,48 @@ +requested_decode_storage_backend=paged +2026-10-11T05:14:15.648214Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:14:15.648245Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:14:15.648353Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:14:15.648444Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:14:15.648447Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:14:15.648449Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:14:16.264220Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:14:16.264798Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=1308622848 representative_snapshot_bytes=218103808 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=218103808 fixed_state_bytes=0 available_ceiling_bytes=30782098150 architecture=sliding-window: 22 layer(s) capped at 512 tokens, 4 global +2026-10-11T05:14:16.275270Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=1308622848 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:14:16.898080Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:14:16.898199Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:14:17.518246Z INFO mlxcel::server::model_provider::model_worker: Model gemma-3-1b-it-4bit loaded in 0.620s (resident after load: 0.00 GB) worker_model_id=gemma-3-1b-it-4bit load_seconds=0.620023327 active_bytes=1256 peak_bytes=1256 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:14:17.522984Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [1] +2026-10-11T05:14:17.523008Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T05:14:17.523298Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 20348273 blocks (26 layers, 32-token blocks) +2026-10-11T05:14:17.523309Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:14:17.523330Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=26 kv_cache_mode_total_layers=26 +2026-10-11T05:14:17.524223Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:14:17.938509Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 414ms prompt_tokens=2 cached_tokens=0 generation_time_ms=414 +2026-10-11T05:14:17.938585Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:14:17.939476Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:14:17.939493Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:14:17.939496Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:14:17.939497Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:14:17.939499Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:14:17.939500Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:14:17.939501Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:14:17.939503Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:14:17.939504Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:14:17.939505Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:14:17.939506Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:14:17.939508Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:14:17.939509Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:14:18.368370Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:14:19.197101Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 827ms prompt_tokens=441 cached_tokens=0 generation_time_ms=827 +2026-10-11T05:14:20.615657Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1392ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1392 +2026-10-11T05:14:20.616135Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1393ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1393 +2026-10-11T05:14:20.616460Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1393ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1393 +2026-10-11T05:14:20.616733Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/441 prompt tokens, total 1393ms prompt_tokens=441 cached_tokens=0 generation_time_ms=1393 +2026-10-11T05:14:25.946394Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5289ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5289 +2026-10-11T05:14:25.947169Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5321ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5321 +2026-10-11T05:14:25.947747Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5322ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5322 +2026-10-11T05:14:25.948252Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5291ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5291 +2026-10-11T05:14:25.948746Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5323ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5323 +2026-10-11T05:14:25.949226Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5323ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5323 +2026-10-11T05:14:25.949695Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5292ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5292 +2026-10-11T05:14:25.950165Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=438/441 prompt tokens, total 5292ms prompt_tokens=441 cached_tokens=438 generation_time_ms=5292 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r1-dense.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r1-dense.log new file mode 100644 index 000000000..61ec33231 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r1-dense.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=dense +2026-10-11T06:24:59.670838Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T06:24:59.670865Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T06:24:59.671155Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T06:24:59.671196Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T06:24:59.671198Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T06:24:59.671200Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T06:25:00.156601Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T06:25:00.158603Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T06:25:00.159278Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T06:25:00.607057Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T06:25:00.607083Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T06:25:01.116359Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.509s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.509256158 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T06:25:01.116417Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T06:25:01.116433Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4) +2026-10-11T06:25:01.117304Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T06:25:01.117348Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T06:25:01.117374Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T06:25:01.118767Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T06:25:34.347254Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 33229ms prompt_tokens=2 cached_tokens=0 generation_time_ms=33229 +2026-10-11T06:25:34.348133Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T06:25:34.348264Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T06:25:34.349588Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T06:25:34.350498Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T06:25:34.350505Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T06:25:34.350508Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T06:25:34.350510Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T06:25:34.350511Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T06:25:34.350512Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T06:25:34.350514Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T06:25:34.350515Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T06:25:34.350516Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T06:25:34.350517Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T06:25:34.350518Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T06:25:34.350520Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T06:25:34.350521Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T06:25:34.794114Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T06:25:50.737784Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3466 prompt tokens, total 15938ms prompt_tokens=3466 cached_tokens=0 generation_time_ms=15938 +2026-10-11T06:26:12.941960Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 21998ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=21998 +2026-10-11T06:26:12.942463Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 21999ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=21999 +2026-10-11T06:26:12.942969Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 21999ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=21999 +2026-10-11T06:26:12.943478Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22000ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22000 +2026-10-11T06:27:22.944212Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69790ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69790 +2026-10-11T06:27:22.944585Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69791ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69791 +2026-10-11T06:27:22.945186Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69794ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69794 +2026-10-11T06:27:22.945469Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69792ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69792 +2026-10-11T06:27:22.945716Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69793ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69793 +2026-10-11T06:27:22.946038Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69794ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69794 +2026-10-11T06:27:22.946192Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69793ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69793 +2026-10-11T06:27:22.946487Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69795ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69795 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r1-paged-a.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r1-paged-a.log new file mode 100644 index 000000000..9ceb08539 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r1-paged-a.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=paged +2026-10-11T06:21:29.537377Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T06:21:29.537407Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T06:21:29.537712Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T06:21:29.537770Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T06:21:29.537773Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T06:21:29.537775Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T06:21:30.022530Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T06:21:30.035255Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T06:21:30.035953Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T06:21:30.486048Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T06:21:30.486264Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T06:21:31.006863Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.521s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.520554821 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T06:21:31.006931Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T06:21:31.006948Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T06:21:31.007790Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T06:21:31.007832Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T06:21:31.007860Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T06:21:31.009242Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T06:22:05.247279Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 34239ms prompt_tokens=2 cached_tokens=0 generation_time_ms=34239 +2026-10-11T06:22:05.248274Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T06:22:05.248430Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T06:22:05.249521Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T06:22:05.250460Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T06:22:05.250467Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T06:22:05.250471Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T06:22:05.250472Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T06:22:05.250473Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T06:22:05.250475Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T06:22:05.250476Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T06:22:05.250477Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T06:22:05.250478Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T06:22:05.250479Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T06:22:05.250481Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T06:22:05.250482Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T06:22:05.250483Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T06:22:05.653517Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T06:22:21.842997Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3466 prompt tokens, total 16183ms prompt_tokens=3466 cached_tokens=0 generation_time_ms=16183 +2026-10-11T06:22:44.052767Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22002ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22002 +2026-10-11T06:22:44.054851Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22004ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22004 +2026-10-11T06:22:44.056778Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22007ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22007 +2026-10-11T06:22:44.058310Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22008ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22008 +2026-10-11T06:23:56.542866Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 72274ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=72274 +2026-10-11T06:23:56.544123Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 72274ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=72274 +2026-10-11T06:23:56.545297Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 72276ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=72276 +2026-10-11T06:23:56.546115Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 72278ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=72278 +2026-10-11T06:23:56.546919Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 72279ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=72279 +2026-10-11T06:23:56.547596Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 72277ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=72277 +2026-10-11T06:23:56.548280Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 72279ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=72279 +2026-10-11T06:23:56.549034Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 72280ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=72280 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r1-paged-b.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r1-paged-b.log new file mode 100644 index 000000000..8d1347c54 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r1-paged-b.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=paged +2026-10-11T06:28:26.010153Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T06:28:26.010182Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T06:28:26.010472Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T06:28:26.010553Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T06:28:26.010557Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T06:28:26.010559Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T06:28:26.479901Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T06:28:26.482019Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T06:28:26.482702Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T06:28:26.929545Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T06:28:26.929962Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T06:28:27.435130Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.505s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.505145088 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T06:28:27.435187Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T06:28:27.435203Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T06:28:27.436068Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T06:28:27.436110Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T06:28:27.436137Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T06:28:27.437485Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T06:29:01.699168Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 34262ms prompt_tokens=2 cached_tokens=0 generation_time_ms=34262 +2026-10-11T06:29:01.700084Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T06:29:01.700241Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T06:29:01.700846Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T06:29:01.701784Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T06:29:01.701792Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T06:29:01.701796Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T06:29:01.701798Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T06:29:01.701799Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T06:29:01.701800Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T06:29:01.701802Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T06:29:01.701803Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T06:29:01.701804Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T06:29:01.701805Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T06:29:01.701807Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T06:29:01.701808Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T06:29:01.701809Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T06:29:02.153727Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T06:29:18.273138Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3466 prompt tokens, total 16111ms prompt_tokens=3466 cached_tokens=0 generation_time_ms=16111 +2026-10-11T06:29:40.353050Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 21872ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=21872 +2026-10-11T06:29:40.354304Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 21873ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=21873 +2026-10-11T06:29:40.355149Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 21873ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=21873 +2026-10-11T06:29:40.355749Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 21874ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=21874 +2026-10-11T06:30:51.166087Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70603ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70603 +2026-10-11T06:30:51.167955Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70606ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70606 +2026-10-11T06:30:51.168927Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70607ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70607 +2026-10-11T06:30:51.169676Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70607ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70607 +2026-10-11T06:30:51.170379Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70609ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70609 +2026-10-11T06:30:51.171110Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70609ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70609 +2026-10-11T06:30:51.171933Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70611ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70611 +2026-10-11T06:30:51.172730Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70610ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70610 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r2-dense.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r2-dense.log new file mode 100644 index 000000000..ca85f1845 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r2-dense.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=dense +2026-10-11T06:31:54.248040Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T06:31:54.248072Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T06:31:54.248371Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T06:31:54.248448Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T06:31:54.248451Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T06:31:54.248453Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T06:31:54.758242Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T06:31:54.770239Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T06:31:54.770942Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T06:31:55.257891Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T06:31:55.257941Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T06:31:55.781533Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.524s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.52357444 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T06:31:55.781596Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T06:31:55.781614Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4) +2026-10-11T06:31:55.782489Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T06:31:55.782532Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T06:31:55.782562Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T06:31:55.783937Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T06:32:29.234799Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 33451ms prompt_tokens=2 cached_tokens=0 generation_time_ms=33451 +2026-10-11T06:32:29.236179Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T06:32:29.236329Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T06:32:29.238423Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T06:32:29.239356Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T06:32:29.239365Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T06:32:29.239368Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T06:32:29.239370Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T06:32:29.239371Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T06:32:29.239372Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T06:32:29.239374Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T06:32:29.239375Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T06:32:29.239376Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T06:32:29.239377Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T06:32:29.239378Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T06:32:29.239380Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T06:32:29.239381Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T06:32:29.326131Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T06:32:45.131805Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3466 prompt tokens, total 15799ms prompt_tokens=3466 cached_tokens=0 generation_time_ms=15799 +2026-10-11T06:33:07.721227Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22386ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22386 +2026-10-11T06:33:07.721591Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22386ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22386 +2026-10-11T06:33:07.721838Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22386ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22386 +2026-10-11T06:33:07.722043Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22386ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22386 +2026-10-11T06:34:17.598570Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69672ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69672 +2026-10-11T06:34:17.599092Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69673ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69673 +2026-10-11T06:34:17.599872Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69673ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69673 +2026-10-11T06:34:17.600237Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69674ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69674 +2026-10-11T06:34:17.600728Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69674ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69674 +2026-10-11T06:34:17.601215Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69675ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69675 +2026-10-11T06:34:17.601705Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69676ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69676 +2026-10-11T06:34:17.602173Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69676ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69676 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r2-paged-a.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r2-paged-a.log new file mode 100644 index 000000000..ab2643921 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r2-paged-a.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=paged +2026-10-11T06:38:49.760330Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T06:38:49.760363Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T06:38:49.760677Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T06:38:49.760755Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T06:38:49.760758Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T06:38:49.760760Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T06:38:50.277080Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T06:38:50.289155Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T06:38:50.289882Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T06:38:50.777395Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T06:38:50.777788Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T06:38:51.286387Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.509s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.508549 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T06:38:51.286444Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T06:38:51.286460Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T06:38:51.287253Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T06:38:51.287292Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T06:38:51.287317Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T06:38:51.288630Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T06:39:26.154149Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 34866ms prompt_tokens=2 cached_tokens=0 generation_time_ms=34866 +2026-10-11T06:39:26.155038Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T06:39:26.155181Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T06:39:26.156304Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T06:39:26.157252Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T06:39:26.157261Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T06:39:26.157264Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T06:39:26.157265Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T06:39:26.157267Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T06:39:26.157268Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T06:39:26.157269Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T06:39:26.157270Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T06:39:26.157272Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T06:39:26.157273Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T06:39:26.157274Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T06:39:26.157275Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T06:39:26.157277Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T06:39:26.895342Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T06:39:42.902474Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3466 prompt tokens, total 15998ms prompt_tokens=3466 cached_tokens=0 generation_time_ms=15998 +2026-10-11T06:40:05.968588Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22851ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22851 +2026-10-11T06:40:05.969616Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22853ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22853 +2026-10-11T06:40:05.970291Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22854ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22854 +2026-10-11T06:40:05.971004Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22854ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22854 +2026-10-11T06:41:15.234237Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69056ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69056 +2026-10-11T06:41:15.236554Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69060ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69060 +2026-10-11T06:41:15.238375Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69060ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69060 +2026-10-11T06:41:15.240447Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69064ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69064 +2026-10-11T06:41:15.242440Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69065ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69065 +2026-10-11T06:41:15.244722Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69066ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69066 +2026-10-11T06:41:15.246754Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69069ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69069 +2026-10-11T06:41:15.248745Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69071ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69071 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r2-paged-b.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r2-paged-b.log new file mode 100644 index 000000000..2ece82663 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r2-paged-b.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=paged +2026-10-11T06:35:20.707922Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T06:35:20.707954Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T06:35:20.708249Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T06:35:20.708343Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T06:35:20.708345Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T06:35:20.708347Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T06:35:21.214765Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T06:35:21.216536Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T06:35:21.217246Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T06:35:21.698786Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T06:35:21.698834Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T06:35:22.174460Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.476s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.475636329 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T06:35:22.174512Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T06:35:22.174527Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T06:35:22.175324Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T06:35:22.175361Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T06:35:22.175385Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T06:35:22.176734Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T06:35:56.931309Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 34755ms prompt_tokens=2 cached_tokens=0 generation_time_ms=34755 +2026-10-11T06:35:56.932481Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T06:35:56.932642Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T06:35:56.933795Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T06:35:56.934673Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T06:35:56.934682Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T06:35:56.934685Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T06:35:56.934687Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T06:35:56.934688Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T06:35:56.934689Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T06:35:56.934691Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T06:35:56.934692Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T06:35:56.934693Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T06:35:56.934695Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T06:35:56.934696Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T06:35:56.934697Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T06:35:56.934699Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T06:35:57.851118Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T06:36:13.900941Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3466 prompt tokens, total 16044ms prompt_tokens=3466 cached_tokens=0 generation_time_ms=16044 +2026-10-11T06:36:36.077043Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 21966ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=21966 +2026-10-11T06:36:36.078087Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 21968ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=21968 +2026-10-11T06:36:36.078814Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 21968ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=21968 +2026-10-11T06:36:36.079556Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 21969ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=21969 +2026-10-11T06:37:46.617797Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70335ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70335 +2026-10-11T06:37:46.620114Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70336ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70336 +2026-10-11T06:37:46.621922Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70339ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70339 +2026-10-11T06:37:46.623564Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70340ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70340 +2026-10-11T06:37:46.625126Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70342ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70342 +2026-10-11T06:37:46.626883Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70344ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70344 +2026-10-11T06:37:46.628437Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70345ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70345 +2026-10-11T06:37:46.630263Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70346ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70346 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r3-dense.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r3-dense.log new file mode 100644 index 000000000..7ad31c06d --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r3-dense.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=dense +2026-10-11T06:49:15.385462Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T06:49:15.385491Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T06:49:15.385795Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T06:49:15.385876Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T06:49:15.385878Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T06:49:15.385880Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T06:49:15.848040Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T06:49:15.859970Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T06:49:15.860657Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T06:49:16.304146Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T06:49:16.304145Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T06:49:16.808731Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.505s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.504541545 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T06:49:16.808794Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T06:49:16.808810Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4) +2026-10-11T06:49:16.809680Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T06:49:16.809722Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T06:49:16.809749Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T06:49:16.811146Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T06:49:52.148171Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 35338ms prompt_tokens=2 cached_tokens=0 generation_time_ms=35338 +2026-10-11T06:49:52.148947Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T06:49:52.149091Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T06:49:52.149941Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T06:49:52.150898Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T06:49:52.150908Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T06:49:52.150911Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T06:49:52.150913Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T06:49:52.150914Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T06:49:52.150915Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T06:49:52.150916Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T06:49:52.150918Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T06:49:52.150919Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T06:49:52.150920Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T06:49:52.150921Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T06:49:52.150923Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T06:49:52.150924Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T06:49:52.526872Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T06:50:08.554394Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3466 prompt tokens, total 16022ms prompt_tokens=3466 cached_tokens=0 generation_time_ms=16022 +2026-10-11T06:50:30.539565Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 21780ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=21780 +2026-10-11T06:50:30.539924Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 21780ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=21780 +2026-10-11T06:50:30.540265Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 21781ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=21781 +2026-10-11T06:50:30.540543Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 21781ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=21781 +2026-10-11T06:51:39.868422Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69124ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69124 +2026-10-11T06:51:39.868971Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69124ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69124 +2026-10-11T06:51:39.869575Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69125ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69125 +2026-10-11T06:51:39.870132Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69126ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69126 +2026-10-11T06:51:39.870663Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69127ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69127 +2026-10-11T06:51:39.871159Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69126ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69126 +2026-10-11T06:51:39.871626Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69126ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69126 +2026-10-11T06:51:39.871950Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69127ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69127 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r3-paged-a.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r3-paged-a.log new file mode 100644 index 000000000..f0c835af0 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r3-paged-a.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=paged +2026-10-11T06:45:46.350375Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T06:45:46.350403Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T06:45:46.350701Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T06:45:46.350742Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T06:45:46.350744Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T06:45:46.350747Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T06:45:46.831356Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T06:45:46.843398Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T06:45:46.844093Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T06:45:47.290539Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T06:45:47.291273Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T06:45:47.808162Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.517s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.516843774 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T06:45:47.808213Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T06:45:47.808227Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T06:45:47.809103Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T06:45:47.809143Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T06:45:47.809169Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T06:45:47.810495Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T06:46:22.414000Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 34604ms prompt_tokens=2 cached_tokens=0 generation_time_ms=34604 +2026-10-11T06:46:22.414891Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T06:46:22.415011Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T06:46:22.415506Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T06:46:22.416407Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T06:46:22.416417Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T06:46:22.416420Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T06:46:22.416422Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T06:46:22.416423Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T06:46:22.416425Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T06:46:22.416426Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T06:46:22.416427Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T06:46:22.416429Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T06:46:22.416430Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T06:46:22.416431Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T06:46:22.416432Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T06:46:22.416434Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T06:46:23.470992Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T06:46:39.578077Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3466 prompt tokens, total 16101ms prompt_tokens=3466 cached_tokens=0 generation_time_ms=16101 +2026-10-11T06:47:01.911409Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22121ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22121 +2026-10-11T06:47:01.912581Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22123ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22123 +2026-10-11T06:47:01.913477Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22123ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22123 +2026-10-11T06:47:01.914111Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22124ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22124 +2026-10-11T06:48:12.230802Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70106ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70106 +2026-10-11T06:48:12.232063Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70107ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70107 +2026-10-11T06:48:12.233019Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70109ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70109 +2026-10-11T06:48:12.234065Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70108ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70108 +2026-10-11T06:48:12.234833Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70109ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70109 +2026-10-11T06:48:12.235799Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70109ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70109 +2026-10-11T06:48:12.236604Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70112ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70112 +2026-10-11T06:48:12.237388Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70112ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70112 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r3-paged-b.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r3-paged-b.log new file mode 100644 index 000000000..0a5f1c7f9 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r3-paged-b.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=paged +2026-10-11T06:42:18.410167Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T06:42:18.410197Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T06:42:18.410497Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T06:42:18.410583Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T06:42:18.410587Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T06:42:18.410589Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T06:42:18.918397Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T06:42:18.929801Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T06:42:18.930495Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T06:42:19.410246Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T06:42:19.410289Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T06:42:19.899488Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.489s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.48919831 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T06:42:19.899555Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T06:42:19.899570Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T06:42:19.900368Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T06:42:19.900404Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T06:42:19.900430Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T06:42:19.901815Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T06:42:53.952273Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 34051ms prompt_tokens=2 cached_tokens=0 generation_time_ms=34051 +2026-10-11T06:42:53.953175Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T06:42:53.953309Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T06:42:53.954063Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T06:42:53.955007Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T06:42:53.955016Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T06:42:53.955020Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T06:42:53.955021Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T06:42:53.955023Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T06:42:53.955024Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T06:42:53.955026Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T06:42:53.955027Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T06:42:53.955028Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T06:42:53.955029Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T06:42:53.955031Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T06:42:53.955032Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T06:42:53.955034Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T06:42:54.512496Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T06:43:10.405581Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3466 prompt tokens, total 15887ms prompt_tokens=3466 cached_tokens=0 generation_time_ms=15887 +2026-10-11T06:43:32.623379Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22012ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22012 +2026-10-11T06:43:32.624572Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22013ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22013 +2026-10-11T06:43:32.625263Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22013ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22013 +2026-10-11T06:43:32.625971Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22014ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22014 +2026-10-11T06:44:43.210831Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70378ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70378 +2026-10-11T06:44:43.213194Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70381ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70381 +2026-10-11T06:44:43.215010Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70383ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70383 +2026-10-11T06:44:43.216847Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70384ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70384 +2026-10-11T06:44:43.218447Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70385ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70385 +2026-10-11T06:44:43.220423Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70389ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70389 +2026-10-11T06:44:43.222235Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70390ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70390 +2026-10-11T06:44:43.224220Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 70392ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=70392 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r4-dense.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r4-dense.log new file mode 100644 index 000000000..55b036217 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r4-dense.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=dense +2026-10-11T06:56:10.045852Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T06:56:10.045885Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T06:56:10.046181Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T06:56:10.046257Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T06:56:10.046259Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T06:56:10.046262Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T06:56:10.563838Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T06:56:10.565931Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T06:56:10.566621Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T06:56:11.051692Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T06:56:11.052432Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T06:56:11.526331Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.474s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.473858481 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T06:56:11.526386Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T06:56:11.526402Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4) +2026-10-11T06:56:11.527208Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T06:56:11.527248Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T06:56:11.527272Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T06:56:11.528604Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T06:56:44.899066Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 33371ms prompt_tokens=2 cached_tokens=0 generation_time_ms=33371 +2026-10-11T06:56:44.899915Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T06:56:44.900063Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T06:56:44.901006Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T06:56:44.901948Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T06:56:44.901957Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T06:56:44.901960Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T06:56:44.901962Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T06:56:44.901963Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T06:56:44.901965Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T06:56:44.901966Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T06:56:44.901967Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T06:56:44.901968Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T06:56:44.901970Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T06:56:44.901971Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T06:56:44.901972Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T06:56:44.901974Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T06:56:45.158276Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T06:57:01.094617Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3466 prompt tokens, total 15931ms prompt_tokens=3466 cached_tokens=0 generation_time_ms=15931 +2026-10-11T06:57:23.026625Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 21726ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=21726 +2026-10-11T06:57:23.027400Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 21727ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=21727 +2026-10-11T06:57:23.027867Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 21727ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=21727 +2026-10-11T06:57:23.028188Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 21728ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=21728 +2026-10-11T06:58:32.330812Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69095ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69095 +2026-10-11T06:58:32.331393Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69097ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69097 +2026-10-11T06:58:32.331961Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69097ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69097 +2026-10-11T06:58:32.332667Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69096ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69096 +2026-10-11T06:58:32.333119Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69097ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69097 +2026-10-11T06:58:32.333450Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69099ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69099 +2026-10-11T06:58:32.333928Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69098ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69098 +2026-10-11T06:58:32.334252Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69099ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69099 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r4-paged-a.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r4-paged-a.log new file mode 100644 index 000000000..e672fb6b7 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r4-paged-a.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=paged +2026-10-11T06:52:42.996670Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T06:52:42.996700Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T06:52:42.996993Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T06:52:42.997050Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T06:52:42.997052Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T06:52:42.997054Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T06:52:43.475573Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T06:52:43.477664Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T06:52:43.478323Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T06:52:43.918612Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T06:52:43.919350Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T06:52:44.422760Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.503s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.503353245 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T06:52:44.422823Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T06:52:44.422840Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T06:52:44.423689Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T06:52:44.423730Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T06:52:44.423754Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T06:52:44.425105Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T06:53:18.206400Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 33782ms prompt_tokens=2 cached_tokens=0 generation_time_ms=33782 +2026-10-11T06:53:18.207227Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T06:53:18.207354Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T06:53:18.208680Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T06:53:18.209498Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T06:53:18.209504Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T06:53:18.209508Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T06:53:18.209509Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T06:53:18.209511Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T06:53:18.209512Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T06:53:18.209513Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T06:53:18.209515Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T06:53:18.209516Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T06:53:18.209517Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T06:53:18.209519Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T06:53:18.209520Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T06:53:18.209522Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T06:53:19.125892Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T06:53:35.154320Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3466 prompt tokens, total 16022ms prompt_tokens=3466 cached_tokens=0 generation_time_ms=16022 +2026-10-11T06:53:57.062839Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 21701ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=21701 +2026-10-11T06:53:57.064282Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 21702ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=21702 +2026-10-11T06:53:57.065011Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 21703ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=21703 +2026-10-11T06:53:57.065775Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 21704ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=21704 +2026-10-11T06:55:06.930156Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69658ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69658 +2026-10-11T06:55:06.931481Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69661ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69661 +2026-10-11T06:55:06.932429Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69661ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69661 +2026-10-11T06:55:06.933436Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69662ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69662 +2026-10-11T06:55:06.934388Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69662ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69662 +2026-10-11T06:55:06.935428Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69664ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69664 +2026-10-11T06:55:06.936321Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69664ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69664 +2026-10-11T06:55:06.937186Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69665ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69665 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r4-paged-b.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r4-paged-b.log new file mode 100644 index 000000000..70fedc0cd --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r4-paged-b.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=paged +2026-10-11T06:59:35.483869Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T06:59:35.483899Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T06:59:35.484195Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T06:59:35.484282Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T06:59:35.484284Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T06:59:35.484287Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T06:59:35.975935Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T06:59:35.987121Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T06:59:35.987836Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T06:59:36.466446Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T06:59:36.466856Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T06:59:36.921246Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.454s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.454352924 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T06:59:36.921297Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T06:59:36.921312Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T06:59:36.922118Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T06:59:36.922157Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T06:59:36.922181Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T06:59:36.923495Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T07:00:11.138533Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 34216ms prompt_tokens=2 cached_tokens=0 generation_time_ms=34216 +2026-10-11T07:00:11.139923Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T07:00:11.140066Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T07:00:11.141312Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T07:00:11.142257Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T07:00:11.142266Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T07:00:11.142269Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T07:00:11.142271Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T07:00:11.142273Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T07:00:11.142274Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T07:00:11.142275Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T07:00:11.142276Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T07:00:11.142278Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T07:00:11.142279Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T07:00:11.142280Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T07:00:11.142282Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T07:00:11.142283Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T07:00:11.634967Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T07:00:27.535168Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3466 prompt tokens, total 15891ms prompt_tokens=3466 cached_tokens=0 generation_time_ms=15891 +2026-10-11T07:00:50.094599Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22351ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22351 +2026-10-11T07:00:50.095704Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22353ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22353 +2026-10-11T07:00:50.096522Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22354ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22354 +2026-10-11T07:00:50.097195Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22353ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22353 +2026-10-11T07:01:58.447464Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 68145ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=68145 +2026-10-11T07:01:58.448692Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 68146ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=68146 +2026-10-11T07:01:58.449780Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 68148ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=68148 +2026-10-11T07:01:58.450945Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 68149ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=68149 +2026-10-11T07:01:58.451765Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 68149ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=68149 +2026-10-11T07:01:58.452580Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 68150ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=68150 +2026-10-11T07:01:58.453161Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 68150ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=68150 +2026-10-11T07:01:58.453964Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 68153ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=68153 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r5-dense.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r5-dense.log new file mode 100644 index 000000000..3d3d4c807 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r5-dense.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=dense +2026-10-11T07:03:01.503100Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T07:03:01.503130Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T07:03:01.503425Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T07:03:01.503492Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T07:03:01.503494Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T07:03:01.503496Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T07:03:01.973237Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T07:03:01.985158Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T07:03:01.985854Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T07:03:02.429918Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T07:03:02.430436Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T07:03:02.952074Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.522s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.521604173 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T07:03:02.952136Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T07:03:02.952151Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4) +2026-10-11T07:03:02.953003Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T07:03:02.953043Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T07:03:02.953068Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T07:03:02.954406Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T07:03:37.085660Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 34132ms prompt_tokens=2 cached_tokens=0 generation_time_ms=34132 +2026-10-11T07:03:37.086439Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T07:03:37.086573Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T07:03:37.087252Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T07:03:37.088158Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T07:03:37.088167Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T07:03:37.088170Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T07:03:37.088172Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T07:03:37.088173Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T07:03:37.088175Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T07:03:37.088176Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T07:03:37.088177Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T07:03:37.088179Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T07:03:37.088180Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T07:03:37.088181Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T07:03:37.088183Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T07:03:37.088184Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T07:03:37.624890Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T07:03:53.488181Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3466 prompt tokens, total 15857ms prompt_tokens=3466 cached_tokens=0 generation_time_ms=15857 +2026-10-11T07:04:16.094473Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22402ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22402 +2026-10-11T07:04:16.094841Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22402ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22402 +2026-10-11T07:04:16.095201Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22403ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22403 +2026-10-11T07:04:16.095416Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22403ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22403 +2026-10-11T07:05:25.892843Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69595ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69595 +2026-10-11T07:05:25.893393Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69596ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69596 +2026-10-11T07:05:25.893924Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69596ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69596 +2026-10-11T07:05:25.894168Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69596ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69596 +2026-10-11T07:05:25.894466Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69597ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69597 +2026-10-11T07:05:25.894701Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69597ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69597 +2026-10-11T07:05:25.895009Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69597ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69597 +2026-10-11T07:05:25.895277Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69597ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69597 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r5-paged-a.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r5-paged-a.log new file mode 100644 index 000000000..b4f2b1e23 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r5-paged-a.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=paged +2026-10-11T07:09:57.480665Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T07:09:57.480694Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T07:09:57.480987Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T07:09:57.481053Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T07:09:57.481055Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T07:09:57.481057Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T07:09:57.950558Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T07:09:57.962440Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T07:09:57.963130Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T07:09:58.407720Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T07:09:58.407737Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T07:09:58.922097Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.514s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.51433462 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T07:09:58.922156Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T07:09:58.922173Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T07:09:58.923038Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T07:09:58.923079Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T07:09:58.923106Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T07:09:58.924476Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T07:10:33.791637Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 34868ms prompt_tokens=2 cached_tokens=0 generation_time_ms=34868 +2026-10-11T07:10:33.792860Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T07:10:33.793101Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T07:10:33.794471Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T07:10:33.795412Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T07:10:33.795420Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T07:10:33.795423Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T07:10:33.795425Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T07:10:33.795426Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T07:10:33.795428Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T07:10:33.795429Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T07:10:33.795430Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T07:10:33.795431Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T07:10:33.795433Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T07:10:33.795434Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T07:10:33.795435Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T07:10:33.795436Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T07:10:34.611868Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T07:10:50.561449Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3466 prompt tokens, total 15941ms prompt_tokens=3466 cached_tokens=0 generation_time_ms=15941 +2026-10-11T07:11:13.287765Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22517ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22517 +2026-10-11T07:11:13.289239Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22518ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22518 +2026-10-11T07:11:13.290063Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22519ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22519 +2026-10-11T07:11:13.290652Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22520ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22520 +2026-10-11T07:12:22.453083Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 68950ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=68950 +2026-10-11T07:12:22.454801Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 68951ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=68951 +2026-10-11T07:12:22.455917Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 68951ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=68951 +2026-10-11T07:12:22.456902Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 68954ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=68954 +2026-10-11T07:12:22.457739Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 68952ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=68952 +2026-10-11T07:12:22.458615Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 68954ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=68954 +2026-10-11T07:12:22.459422Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 68955ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=68955 +2026-10-11T07:12:22.460214Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 68956ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=68956 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r5-paged-b.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r5-paged-b.log new file mode 100644 index 000000000..879a55a9a --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p4096-r5-paged-b.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=paged +2026-10-11T07:06:28.991472Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T07:06:28.991503Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T07:06:28.991808Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T07:06:28.991893Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T07:06:28.991896Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T07:06:28.991898Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T07:06:29.488199Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T07:06:29.500826Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T07:06:29.501580Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T07:06:29.988356Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T07:06:29.988996Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T07:06:30.475468Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.486s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.486423612 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T07:06:30.475525Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T07:06:30.475543Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T07:06:30.476330Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T07:06:30.476367Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T07:06:30.476392Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T07:06:30.477733Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T07:07:05.043615Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 34566ms prompt_tokens=2 cached_tokens=0 generation_time_ms=34566 +2026-10-11T07:07:05.044287Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T07:07:05.044414Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T07:07:05.044932Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T07:07:05.045784Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T07:07:05.045791Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T07:07:05.045795Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T07:07:05.045797Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T07:07:05.045798Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T07:07:05.045799Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T07:07:05.045801Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T07:07:05.045802Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T07:07:05.045803Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T07:07:05.045804Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T07:07:05.045805Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T07:07:05.045807Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T07:07:05.045808Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T07:07:06.126632Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T07:07:22.076589Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/3466 prompt tokens, total 15944ms prompt_tokens=3466 cached_tokens=0 generation_time_ms=15944 +2026-10-11T07:07:44.332149Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22048ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22048 +2026-10-11T07:07:44.333509Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22050ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22050 +2026-10-11T07:07:44.334338Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22050ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22050 +2026-10-11T07:07:44.334956Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 22051ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=22051 +2026-10-11T07:08:54.306989Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69764ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69764 +2026-10-11T07:08:54.308619Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69766ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69766 +2026-10-11T07:08:54.309594Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69767ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69767 +2026-10-11T07:08:54.310489Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69767ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69767 +2026-10-11T07:08:54.311340Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69769ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69769 +2026-10-11T07:08:54.312468Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69770ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69770 +2026-10-11T07:08:54.313435Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69771ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69771 +2026-10-11T07:08:54.314215Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=3465/3466 prompt tokens, total 69771ms prompt_tokens=3466 cached_tokens=3465 generation_time_ms=69771 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r1-dense.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r1-dense.log new file mode 100644 index 000000000..c066ff100 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r1-dense.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=dense +2026-10-11T05:38:32.480193Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:38:32.480224Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:38:32.480559Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:38:32.480650Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:38:32.480653Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:38:32.480655Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:38:32.931942Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:38:32.943445Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T05:38:32.944123Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:38:33.389649Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:38:33.389681Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:38:33.929499Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.540s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.539793436 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:38:33.929562Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T05:38:33.929578Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4) +2026-10-11T05:38:33.930415Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T05:38:33.930453Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:38:33.930517Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T05:38:33.931891Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:39:08.505708Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 34574ms prompt_tokens=2 cached_tokens=0 generation_time_ms=34574 +2026-10-11T05:39:08.506492Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:39:08.506662Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T05:39:08.507346Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T05:39:08.508429Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:39:08.508442Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:39:08.508446Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:39:08.508447Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:39:08.508449Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:39:08.508450Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:39:08.508451Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:39:08.508452Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:39:08.508454Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:39:08.508455Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:39:08.508456Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:39:08.508457Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:39:08.508459Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:39:08.596387Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:39:19.373344Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/442 prompt tokens, total 10775ms prompt_tokens=442 cached_tokens=0 generation_time_ms=10775 +2026-10-11T05:39:36.350868Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16798ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16798 +2026-10-11T05:39:36.351232Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16798ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16798 +2026-10-11T05:39:36.351434Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16799ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16799 +2026-10-11T05:39:36.351729Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16799ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16799 +2026-10-11T05:40:33.425324Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56884ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56884 +2026-10-11T05:40:33.426060Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56884ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56884 +2026-10-11T05:40:33.426396Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56885ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56885 +2026-10-11T05:40:33.426854Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56886ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56886 +2026-10-11T05:40:33.427247Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56887ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56887 +2026-10-11T05:40:33.427680Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56887ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56887 +2026-10-11T05:40:33.428091Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56886ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56886 +2026-10-11T05:40:33.428482Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56888ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56888 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r1-paged-a.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r1-paged-a.log new file mode 100644 index 000000000..7459094f6 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r1-paged-a.log @@ -0,0 +1,51 @@ +requested_decode_storage_backend=paged +2026-10-11T05:35:10.085459Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:35:10.085490Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:35:10.085805Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:35:10.085870Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:35:10.085873Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:35:10.085875Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:35:10.555620Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:35:10.567894Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T05:35:10.568589Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:35:11.013810Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:35:11.014223Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:35:11.552086Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.538s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.53782656 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:35:11.552142Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T05:35:11.552156Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T05:35:11.553004Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T05:35:11.553045Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:35:11.553069Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T05:35:11.554423Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +Compiling CUDA kernels (first run on this host; cached for later runs)... +2026-10-11T05:35:48.122191Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 36568ms prompt_tokens=2 cached_tokens=0 generation_time_ms=36568 +2026-10-11T05:35:48.123147Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:35:48.123340Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T05:35:48.124011Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T05:35:48.125256Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:35:48.125262Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:35:48.125305Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:35:48.125306Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:35:48.125308Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:35:48.125309Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:35:48.125310Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:35:48.125312Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:35:48.125313Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:35:48.125314Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:35:48.125315Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:35:48.125317Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:35:48.125318Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:35:48.254417Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:36:08.063494Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/442 prompt tokens, total 19807ms prompt_tokens=442 cached_tokens=0 generation_time_ms=19807 +2026-10-11T05:36:24.486265Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16232ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16232 +2026-10-11T05:36:24.486975Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16232ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16232 +2026-10-11T05:36:24.487659Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16233ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16233 +2026-10-11T05:36:24.488346Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16233ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16233 +2026-10-11T05:37:29.200680Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 64528ms prompt_tokens=442 cached_tokens=441 generation_time_ms=64528 +2026-10-11T05:37:29.201554Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 64529ms prompt_tokens=442 cached_tokens=441 generation_time_ms=64529 +2026-10-11T05:37:29.202038Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 64529ms prompt_tokens=442 cached_tokens=441 generation_time_ms=64529 +2026-10-11T05:37:29.202343Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 64529ms prompt_tokens=442 cached_tokens=441 generation_time_ms=64529 +2026-10-11T05:37:29.202728Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 64529ms prompt_tokens=442 cached_tokens=441 generation_time_ms=64529 +2026-10-11T05:37:29.202987Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 64530ms prompt_tokens=442 cached_tokens=441 generation_time_ms=64530 +2026-10-11T05:37:29.203488Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 64532ms prompt_tokens=442 cached_tokens=441 generation_time_ms=64532 +2026-10-11T05:37:29.203777Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 64532ms prompt_tokens=442 cached_tokens=441 generation_time_ms=64532 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r1-paged-b.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r1-paged-b.log new file mode 100644 index 000000000..b42df1bda --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r1-paged-b.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=paged +2026-10-11T05:41:36.573964Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:41:36.573990Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:41:36.574282Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:41:36.574348Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:41:36.574351Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:41:36.574353Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:41:37.064908Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:41:37.076109Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T05:41:37.076781Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:41:37.521418Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:41:37.522106Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:41:38.047810Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.526s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.52564145 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:41:38.047869Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T05:41:38.047887Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T05:41:38.048745Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T05:41:38.048787Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:41:38.048814Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T05:41:38.050214Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:42:12.276701Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 34227ms prompt_tokens=2 cached_tokens=0 generation_time_ms=34227 +2026-10-11T05:42:12.277525Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:42:12.277672Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T05:42:12.278820Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T05:42:12.279782Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:42:12.279790Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:42:12.279793Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:42:12.279795Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:42:12.279796Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:42:12.279797Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:42:12.279799Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:42:12.279800Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:42:12.279801Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:42:12.279802Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:42:12.279804Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:42:12.279805Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:42:12.279806Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:42:12.698271Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:42:23.547443Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/442 prompt tokens, total 10848ms prompt_tokens=442 cached_tokens=0 generation_time_ms=10848 +2026-10-11T05:42:39.393411Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 15662ms prompt_tokens=442 cached_tokens=441 generation_time_ms=15662 +2026-10-11T05:42:39.393876Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 15663ms prompt_tokens=442 cached_tokens=441 generation_time_ms=15663 +2026-10-11T05:42:39.394284Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 15664ms prompt_tokens=442 cached_tokens=441 generation_time_ms=15664 +2026-10-11T05:42:39.394629Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 15664ms prompt_tokens=442 cached_tokens=441 generation_time_ms=15664 +2026-10-11T05:43:37.769091Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 58192ms prompt_tokens=442 cached_tokens=441 generation_time_ms=58192 +2026-10-11T05:43:37.769661Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 58192ms prompt_tokens=442 cached_tokens=441 generation_time_ms=58192 +2026-10-11T05:43:37.770133Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 58194ms prompt_tokens=442 cached_tokens=441 generation_time_ms=58194 +2026-10-11T05:43:37.770569Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 58193ms prompt_tokens=442 cached_tokens=441 generation_time_ms=58193 +2026-10-11T05:43:37.770820Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 58193ms prompt_tokens=442 cached_tokens=441 generation_time_ms=58193 +2026-10-11T05:43:37.771187Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 58193ms prompt_tokens=442 cached_tokens=441 generation_time_ms=58193 +2026-10-11T05:43:37.771542Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 58193ms prompt_tokens=442 cached_tokens=441 generation_time_ms=58193 +2026-10-11T05:43:37.771804Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 58195ms prompt_tokens=442 cached_tokens=441 generation_time_ms=58195 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r2-dense.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r2-dense.log new file mode 100644 index 000000000..ed794711f --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r2-dense.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=dense +2026-10-11T05:44:40.832027Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:44:40.832055Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:44:40.832352Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:44:40.832410Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:44:40.832413Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:44:40.832415Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:44:41.311115Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:44:41.313211Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T05:44:41.313893Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:44:41.758992Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:44:41.759415Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:44:42.291836Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.532s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.532387858 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:44:42.291902Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T05:44:42.291917Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4) +2026-10-11T05:44:42.292776Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T05:44:42.292819Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:44:42.292845Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T05:44:42.294179Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:45:16.958291Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 34665ms prompt_tokens=2 cached_tokens=0 generation_time_ms=34665 +2026-10-11T05:45:16.959119Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:45:16.959247Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T05:45:16.959867Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T05:45:16.960787Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:45:16.960795Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:45:16.960799Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:45:16.960800Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:45:16.960802Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:45:16.960803Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:45:16.960804Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:45:16.960805Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:45:16.960807Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:45:16.960808Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:45:16.960809Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:45:16.960810Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:45:16.960812Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:45:18.010134Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:45:28.891723Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/442 prompt tokens, total 10880ms prompt_tokens=442 cached_tokens=0 generation_time_ms=10880 +2026-10-11T05:45:45.495946Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16424ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16424 +2026-10-11T05:45:45.496353Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16424ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16424 +2026-10-11T05:45:45.496642Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16424ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16424 +2026-10-11T05:45:45.497021Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16425ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16425 +2026-10-11T05:46:42.081645Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56402ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56402 +2026-10-11T05:46:42.082287Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56402ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56402 +2026-10-11T05:46:42.082792Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56403ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56403 +2026-10-11T05:46:42.083225Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56402ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56402 +2026-10-11T05:46:42.083736Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56402ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56402 +2026-10-11T05:46:42.084166Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56404ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56404 +2026-10-11T05:46:42.084712Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56405ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56405 +2026-10-11T05:46:42.084989Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56406ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56406 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r2-paged-a.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r2-paged-a.log new file mode 100644 index 000000000..6e9206cd1 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r2-paged-a.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=paged +2026-10-11T05:50:48.265373Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:50:48.265405Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:50:48.265711Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:50:48.265791Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:50:48.265794Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:50:48.265796Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:50:48.775026Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:50:48.786142Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T05:50:48.786874Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:50:49.264075Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:50:49.264081Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:50:49.767965Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.504s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.5038398 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:50:49.768022Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T05:50:49.768035Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T05:50:49.768874Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T05:50:49.768915Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:50:49.768940Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T05:50:49.770270Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:51:24.202545Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 34433ms prompt_tokens=2 cached_tokens=0 generation_time_ms=34433 +2026-10-11T05:51:24.203430Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:51:24.203582Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T05:51:24.204245Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T05:51:24.205178Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:51:24.205191Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:51:24.205194Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:51:24.205196Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:51:24.205197Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:51:24.205199Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:51:24.205200Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:51:24.205201Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:51:24.205203Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:51:24.205204Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:51:24.205205Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:51:24.205206Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:51:24.205208Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:51:24.377908Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:51:35.209040Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/442 prompt tokens, total 10829ms prompt_tokens=442 cached_tokens=0 generation_time_ms=10829 +2026-10-11T05:51:51.996010Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16606ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16606 +2026-10-11T05:51:51.996653Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16606ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16606 +2026-10-11T05:51:51.997213Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16607ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16607 +2026-10-11T05:51:51.997922Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16607ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16607 +2026-10-11T05:52:48.602553Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56420ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56420 +2026-10-11T05:52:48.603551Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56422ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56422 +2026-10-11T05:52:48.604144Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56423ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56423 +2026-10-11T05:52:48.604733Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56423ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56423 +2026-10-11T05:52:48.605414Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56424ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56424 +2026-10-11T05:52:48.606062Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56424ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56424 +2026-10-11T05:52:48.606937Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56426ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56426 +2026-10-11T05:52:48.607511Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56425ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56425 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r2-paged-b.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r2-paged-b.log new file mode 100644 index 000000000..941726dc0 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r2-paged-b.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=paged +2026-10-11T05:47:45.211955Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:47:45.211982Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:47:45.212279Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:47:45.212343Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:47:45.212345Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:47:45.212347Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:47:45.687947Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:47:45.698989Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T05:47:45.699662Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:47:46.154351Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:47:46.154413Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:47:46.659224Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.505s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.504835151 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:47:46.659274Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T05:47:46.659289Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T05:47:46.660128Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T05:47:46.660171Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:47:46.660197Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T05:47:46.661562Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:48:19.402159Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 32741ms prompt_tokens=2 cached_tokens=0 generation_time_ms=32741 +2026-10-11T05:48:19.402844Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:48:19.402974Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T05:48:19.403494Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T05:48:19.404394Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:48:19.404407Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:48:19.404411Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:48:19.404413Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:48:19.404414Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:48:19.404415Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:48:19.404417Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:48:19.404418Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:48:19.404419Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:48:19.404420Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:48:19.404422Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:48:19.404423Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:48:19.404425Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:48:20.457649Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:48:31.229892Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/442 prompt tokens, total 10771ms prompt_tokens=442 cached_tokens=0 generation_time_ms=10771 +2026-10-11T05:48:47.894785Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16483ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16483 +2026-10-11T05:48:47.895154Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16483ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16483 +2026-10-11T05:48:47.895517Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16483ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16483 +2026-10-11T05:48:47.895847Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16484ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16484 +2026-10-11T05:49:45.144950Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57071ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57071 +2026-10-11T05:49:45.145891Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57072ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57072 +2026-10-11T05:49:45.146375Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57072ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57072 +2026-10-11T05:49:45.146657Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57073ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57073 +2026-10-11T05:49:45.146924Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57073ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57073 +2026-10-11T05:49:45.147264Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57073ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57073 +2026-10-11T05:49:45.147598Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57074ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57074 +2026-10-11T05:49:45.147767Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57074ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57074 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r3-dense.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r3-dense.log new file mode 100644 index 000000000..d751846b8 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r3-dense.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=dense +2026-10-11T06:00:01.051415Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T06:00:01.051445Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T06:00:01.051747Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T06:00:01.051789Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T06:00:01.051791Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T06:00:01.051793Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T06:00:01.538930Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T06:00:01.549945Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T06:00:01.550617Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T06:00:01.998851Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T06:00:01.998902Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T06:00:02.521274Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.522s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.5223848 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T06:00:02.521330Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T06:00:02.521345Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4) +2026-10-11T06:00:02.522212Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T06:00:02.522255Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T06:00:02.522280Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T06:00:02.523633Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T06:00:36.497638Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 33975ms prompt_tokens=2 cached_tokens=0 generation_time_ms=33975 +2026-10-11T06:00:36.498577Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T06:00:36.498721Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T06:00:36.499807Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T06:00:36.500665Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T06:00:36.500672Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T06:00:36.500675Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T06:00:36.500677Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T06:00:36.500678Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T06:00:36.500680Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T06:00:36.500682Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T06:00:36.500683Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T06:00:36.500684Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T06:00:36.500686Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T06:00:36.500687Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T06:00:36.500688Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T06:00:36.500690Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T06:00:37.159376Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T06:00:48.006153Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/442 prompt tokens, total 10845ms prompt_tokens=442 cached_tokens=0 generation_time_ms=10845 +2026-10-11T06:01:04.431998Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16243ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16243 +2026-10-11T06:01:04.432474Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16244ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16244 +2026-10-11T06:01:04.432767Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16244ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16244 +2026-10-11T06:01:04.433064Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16244ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16244 +2026-10-11T06:02:03.354596Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 58738ms prompt_tokens=442 cached_tokens=441 generation_time_ms=58738 +2026-10-11T06:02:03.355055Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 58737ms prompt_tokens=442 cached_tokens=441 generation_time_ms=58737 +2026-10-11T06:02:03.355402Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 58738ms prompt_tokens=442 cached_tokens=441 generation_time_ms=58738 +2026-10-11T06:02:03.355872Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 58737ms prompt_tokens=442 cached_tokens=441 generation_time_ms=58737 +2026-10-11T06:02:03.356285Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 58739ms prompt_tokens=442 cached_tokens=441 generation_time_ms=58739 +2026-10-11T06:02:03.356677Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 58739ms prompt_tokens=442 cached_tokens=441 generation_time_ms=58739 +2026-10-11T06:02:03.356902Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 58738ms prompt_tokens=442 cached_tokens=441 generation_time_ms=58738 +2026-10-11T06:02:03.357296Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 58740ms prompt_tokens=442 cached_tokens=441 generation_time_ms=58740 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r3-paged-a.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r3-paged-a.log new file mode 100644 index 000000000..1818d80e6 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r3-paged-a.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=paged +2026-10-11T05:56:56.654851Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:56:56.654881Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:56:56.655178Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:56:56.655254Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:56:56.655256Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:56:56.655258Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:56:57.165336Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:56:57.176366Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T05:56:57.177061Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:56:57.657773Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:56:57.658428Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:56:58.146765Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.488s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.488304338 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:56:58.146823Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T05:56:58.146840Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T05:56:58.147632Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T05:56:58.147669Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:56:58.147694Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T05:56:58.149013Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:57:32.672121Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 34524ms prompt_tokens=2 cached_tokens=0 generation_time_ms=34524 +2026-10-11T05:57:32.672862Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:57:32.672996Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T05:57:32.674150Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T05:57:32.675104Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:57:32.675115Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:57:32.675119Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:57:32.675120Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:57:32.675122Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:57:32.675123Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:57:32.675124Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:57:32.675126Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:57:32.675127Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:57:32.675128Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:57:32.675129Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:57:32.675131Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:57:32.675132Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:57:32.775604Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:57:43.740739Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/442 prompt tokens, total 10963ms prompt_tokens=442 cached_tokens=0 generation_time_ms=10963 +2026-10-11T05:58:00.263629Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16338ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16338 +2026-10-11T05:58:00.264092Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16339ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16339 +2026-10-11T05:58:00.264483Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16339ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16339 +2026-10-11T05:58:00.264726Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16339ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16339 +2026-10-11T05:58:57.973035Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57526ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57526 +2026-10-11T05:58:57.973974Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57527ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57527 +2026-10-11T05:58:57.974496Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57528ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57528 +2026-10-11T05:58:57.975051Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57529ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57529 +2026-10-11T05:58:57.975401Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57529ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57529 +2026-10-11T05:58:57.975745Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57529ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57529 +2026-10-11T05:58:57.975995Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57528ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57528 +2026-10-11T05:58:57.976306Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57528ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57528 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r3-paged-b.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r3-paged-b.log new file mode 100644 index 000000000..f27e5bf40 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r3-paged-b.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=paged +2026-10-11T05:53:51.675866Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T05:53:51.675895Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T05:53:51.676193Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T05:53:51.676256Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T05:53:51.676258Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T05:53:51.676260Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T05:53:52.163266Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T05:53:52.174489Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T05:53:52.175187Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T05:53:52.624834Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T05:53:52.625362Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T05:53:53.164476Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.539s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.539083378 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T05:53:53.164551Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T05:53:53.164567Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T05:53:53.165422Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T05:53:53.165460Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T05:53:53.165487Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T05:53:53.166882Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T05:54:27.756489Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 34590ms prompt_tokens=2 cached_tokens=0 generation_time_ms=34590 +2026-10-11T05:54:27.757400Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T05:54:27.757547Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T05:54:27.758457Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T05:54:27.759375Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T05:54:27.759388Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T05:54:27.759391Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T05:54:27.759393Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T05:54:27.759394Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T05:54:27.759396Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T05:54:27.759397Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T05:54:27.759398Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T05:54:27.759400Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T05:54:27.759401Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T05:54:27.759402Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T05:54:27.759403Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T05:54:27.759405Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T05:54:27.838211Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T05:54:38.629548Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/442 prompt tokens, total 10789ms prompt_tokens=442 cached_tokens=0 generation_time_ms=10789 +2026-10-11T05:54:55.193469Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16382ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16382 +2026-10-11T05:54:55.193987Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16382ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16382 +2026-10-11T05:54:55.194540Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16382ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16382 +2026-10-11T05:54:55.194998Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16383ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16383 +2026-10-11T05:55:53.664837Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 58289ms prompt_tokens=442 cached_tokens=441 generation_time_ms=58289 +2026-10-11T05:55:53.665604Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 58289ms prompt_tokens=442 cached_tokens=441 generation_time_ms=58289 +2026-10-11T05:55:53.666211Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 58290ms prompt_tokens=442 cached_tokens=441 generation_time_ms=58290 +2026-10-11T05:55:53.667010Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 58292ms prompt_tokens=442 cached_tokens=441 generation_time_ms=58292 +2026-10-11T05:55:53.667519Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 58292ms prompt_tokens=442 cached_tokens=441 generation_time_ms=58292 +2026-10-11T05:55:53.667957Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 58291ms prompt_tokens=442 cached_tokens=441 generation_time_ms=58291 +2026-10-11T05:55:53.668614Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 58292ms prompt_tokens=442 cached_tokens=441 generation_time_ms=58292 +2026-10-11T05:55:53.669001Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 58293ms prompt_tokens=442 cached_tokens=441 generation_time_ms=58293 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r4-dense.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r4-dense.log new file mode 100644 index 000000000..8e43ffa98 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r4-dense.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=dense +2026-10-11T06:06:08.649845Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T06:06:08.649876Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T06:06:08.650172Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T06:06:08.650262Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T06:06:08.650265Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T06:06:08.650267Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T06:06:09.133488Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T06:06:09.146610Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T06:06:09.147313Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T06:06:09.597715Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T06:06:09.597977Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T06:06:10.115521Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.518s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.517520163 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T06:06:10.115586Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T06:06:10.115602Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4) +2026-10-11T06:06:10.116474Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T06:06:10.116513Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T06:06:10.116541Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T06:06:10.117863Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T06:06:44.375204Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 34258ms prompt_tokens=2 cached_tokens=0 generation_time_ms=34258 +2026-10-11T06:06:44.375362Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T06:06:44.375493Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T06:06:44.376311Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T06:06:44.377220Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T06:06:44.377228Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T06:06:44.377231Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T06:06:44.377233Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T06:06:44.377234Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T06:06:44.377235Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T06:06:44.377236Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T06:06:44.377237Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T06:06:44.377239Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T06:06:44.377240Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T06:06:44.377241Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T06:06:44.377242Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T06:06:44.377244Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T06:06:44.773052Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T06:06:55.502273Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/442 prompt tokens, total 10727ms prompt_tokens=442 cached_tokens=0 generation_time_ms=10727 +2026-10-11T06:07:12.171951Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16486ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16486 +2026-10-11T06:07:12.172392Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16487ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16487 +2026-10-11T06:07:12.172692Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16487ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16487 +2026-10-11T06:07:12.173111Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16487ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16487 +2026-10-11T06:08:09.531438Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57175ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57175 +2026-10-11T06:08:09.531967Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57175ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57175 +2026-10-11T06:08:09.532358Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57175ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57175 +2026-10-11T06:08:09.532947Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57175ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57175 +2026-10-11T06:08:09.533270Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57175ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57175 +2026-10-11T06:08:09.533664Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57175ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57175 +2026-10-11T06:08:09.534006Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57176ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57176 +2026-10-11T06:08:09.534270Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57178ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57178 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r4-paged-a.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r4-paged-a.log new file mode 100644 index 000000000..757a0b71a --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r4-paged-a.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=paged +2026-10-11T06:03:06.511052Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T06:03:06.511082Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T06:03:06.511376Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T06:03:06.511458Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T06:03:06.511461Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T06:03:06.511463Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T06:03:07.002096Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T06:03:07.014044Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T06:03:07.014740Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T06:03:07.467406Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T06:03:07.467841Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T06:03:07.977859Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.510s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.509957641 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T06:03:07.977915Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T06:03:07.977931Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T06:03:07.978770Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T06:03:07.978811Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T06:03:07.978836Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T06:03:07.980185Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T06:03:40.442593Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 32463ms prompt_tokens=2 cached_tokens=0 generation_time_ms=32463 +2026-10-11T06:03:40.443480Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T06:03:40.443640Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T06:03:40.444432Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T06:03:40.445286Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T06:03:40.445294Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T06:03:40.445297Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T06:03:40.445299Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T06:03:40.445301Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T06:03:40.445302Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T06:03:40.445303Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T06:03:40.445304Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T06:03:40.445306Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T06:03:40.445307Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T06:03:40.445309Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T06:03:40.445310Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T06:03:40.445312Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T06:03:40.652564Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T06:03:51.607564Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/442 prompt tokens, total 10953ms prompt_tokens=442 cached_tokens=0 generation_time_ms=10953 +2026-10-11T06:04:08.145118Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16353ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16353 +2026-10-11T06:04:08.145594Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16354ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16354 +2026-10-11T06:04:08.145887Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16354ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16354 +2026-10-11T06:04:08.146235Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16355ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16355 +2026-10-11T06:05:05.546842Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57222ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57222 +2026-10-11T06:05:05.547963Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57223ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57223 +2026-10-11T06:05:05.548726Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57223ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57223 +2026-10-11T06:05:05.549243Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57224ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57224 +2026-10-11T06:05:05.549908Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57224ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57224 +2026-10-11T06:05:05.550543Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57225ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57225 +2026-10-11T06:05:05.551223Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57227ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57227 +2026-10-11T06:05:05.551782Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57227ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57227 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r4-paged-b.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r4-paged-b.log new file mode 100644 index 000000000..043718678 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r4-paged-b.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=paged +2026-10-11T06:09:12.576175Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T06:09:12.576205Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T06:09:12.576496Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T06:09:12.576564Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T06:09:12.576568Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T06:09:12.576570Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T06:09:13.064495Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T06:09:13.066587Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T06:09:13.067251Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T06:09:13.525999Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T06:09:13.526769Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T06:09:14.046957Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.520s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.520148743 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T06:09:14.047017Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T06:09:14.047033Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T06:09:14.047913Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T06:09:14.047959Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T06:09:14.047986Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T06:09:14.049392Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T06:09:48.853788Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 34805ms prompt_tokens=2 cached_tokens=0 generation_time_ms=34805 +2026-10-11T06:09:48.854924Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T06:09:48.855069Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T06:09:48.856243Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T06:09:48.857178Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T06:09:48.857186Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T06:09:48.857189Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T06:09:48.857191Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T06:09:48.857192Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T06:09:48.857193Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T06:09:48.857194Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T06:09:48.857196Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T06:09:48.857197Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T06:09:48.857198Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T06:09:48.857199Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T06:09:48.857201Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T06:09:48.857202Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T06:09:49.729234Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T06:10:00.567374Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/442 prompt tokens, total 10836ms prompt_tokens=442 cached_tokens=0 generation_time_ms=10836 +2026-10-11T06:10:17.234890Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16485ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16485 +2026-10-11T06:10:17.235523Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16485ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16485 +2026-10-11T06:10:17.236226Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16486ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16486 +2026-10-11T06:10:17.236907Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16487ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16487 +2026-10-11T06:11:14.971495Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57554ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57554 +2026-10-11T06:11:14.972265Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57553ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57553 +2026-10-11T06:11:14.973202Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57556ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57556 +2026-10-11T06:11:14.973944Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57555ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57555 +2026-10-11T06:11:14.974607Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57556ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57556 +2026-10-11T06:11:14.975251Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57557ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57557 +2026-10-11T06:11:14.975777Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57556ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57556 +2026-10-11T06:11:14.976370Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57558ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57558 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r5-dense.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r5-dense.log new file mode 100644 index 000000000..a20f5a35d --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r5-dense.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=dense +2026-10-11T06:12:18.097321Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T06:12:18.097351Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T06:12:18.097664Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T06:12:18.097738Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T06:12:18.097740Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T06:12:18.097742Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T06:12:18.606327Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T06:12:18.617582Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T06:12:18.618282Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T06:12:19.096608Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T06:12:19.097107Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T06:12:19.581328Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.484s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.48418606 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T06:12:19.581381Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T06:12:19.581395Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4) +2026-10-11T06:12:19.582193Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T06:12:19.582232Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T06:12:19.582257Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T06:12:19.583590Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T06:12:53.540019Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 33957ms prompt_tokens=2 cached_tokens=0 generation_time_ms=33957 +2026-10-11T06:12:53.540968Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T06:12:53.541115Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T06:12:53.542073Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T06:12:53.542931Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T06:12:53.542939Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T06:12:53.542942Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T06:12:53.542944Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T06:12:53.542945Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T06:12:53.542947Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T06:12:53.542948Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T06:12:53.542949Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T06:12:53.542951Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T06:12:53.542952Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T06:12:53.542953Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T06:12:53.542954Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T06:12:53.542956Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T06:12:54.217286Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T06:13:05.055157Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/442 prompt tokens, total 10836ms prompt_tokens=442 cached_tokens=0 generation_time_ms=10836 +2026-10-11T06:13:21.353802Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16118ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16118 +2026-10-11T06:13:21.354403Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16117ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16117 +2026-10-11T06:13:21.354898Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16118ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16118 +2026-10-11T06:13:21.355339Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16118ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16118 +2026-10-11T06:14:19.003785Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57467ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57467 +2026-10-11T06:14:19.004410Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57467ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57467 +2026-10-11T06:14:19.005023Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57468ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57468 +2026-10-11T06:14:19.005324Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57467ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57467 +2026-10-11T06:14:19.005726Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57467ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57467 +2026-10-11T06:14:19.005975Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57468ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57468 +2026-10-11T06:14:19.006338Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57470ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57470 +2026-10-11T06:14:19.006570Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57470ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57470 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r5-paged-a.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r5-paged-a.log new file mode 100644 index 000000000..794c1b822 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r5-paged-a.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=paged +2026-10-11T06:18:25.072402Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T06:18:25.072431Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T06:18:25.072742Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T06:18:25.072821Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T06:18:25.072823Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T06:18:25.072825Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T06:18:25.581127Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T06:18:25.592818Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T06:18:25.593505Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T06:18:26.076392Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T06:18:26.076850Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T06:18:26.551596Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.475s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.474705093 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T06:18:26.551649Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T06:18:26.551666Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T06:18:26.552457Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T06:18:26.552494Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T06:18:26.552517Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T06:18:26.553821Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T06:19:00.979268Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 34426ms prompt_tokens=2 cached_tokens=0 generation_time_ms=34426 +2026-10-11T06:19:00.979910Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T06:19:00.980044Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T06:19:00.981685Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T06:19:00.982640Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T06:19:00.982649Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T06:19:00.982652Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T06:19:00.982653Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T06:19:00.982655Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T06:19:00.982656Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T06:19:00.982657Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T06:19:00.982659Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T06:19:00.982660Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T06:19:00.982661Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T06:19:00.982662Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T06:19:00.982663Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T06:19:00.982665Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T06:19:01.194170Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T06:19:12.071295Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/442 prompt tokens, total 10876ms prompt_tokens=442 cached_tokens=0 generation_time_ms=10876 +2026-10-11T06:19:28.824121Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16569ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16569 +2026-10-11T06:19:28.824761Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16569ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16569 +2026-10-11T06:19:28.825173Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16570ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16570 +2026-10-11T06:19:28.825427Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16571ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16571 +2026-10-11T06:20:26.443965Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57438ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57438 +2026-10-11T06:20:26.444826Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57440ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57440 +2026-10-11T06:20:26.445311Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57439ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57439 +2026-10-11T06:20:26.445710Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57440ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57440 +2026-10-11T06:20:26.446020Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57441ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57441 +2026-10-11T06:20:26.446330Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57441ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57441 +2026-10-11T06:20:26.446646Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57442ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57442 +2026-10-11T06:20:26.446900Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 57441ms prompt_tokens=442 cached_tokens=441 generation_time_ms=57441 diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r5-paged-b.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r5-paged-b.log new file mode 100644 index 000000000..baad648b0 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/server-llama4-p512-r5-paged-b.log @@ -0,0 +1,50 @@ +requested_decode_storage_backend=paged +2026-10-11T06:15:22.049404Z WARN mlxcel::server::startup: CORS is set to allow all origins ('*') and no API key is set; this can be a security risk (cross-origin attacks). Set --api-key, or narrow --cors-origins / --allowed-origins +2026-10-11T06:15:22.049433Z INFO mlxcel::server::startup: effective KV cache mode kv_cache_mode=fp16 kv_bits=0 +2026-10-11T06:15:22.049738Z INFO mlxcel::server::startup: resolved context and batch geometry (0 = the checkpoint's own trained context) ctx_size=0 ctx_size_per_slot=0 context_slots=8 kv_unified=false n_parallel=8 prefill_chunk_size=2048 max_kv_size=None +2026-10-11T06:15:22.049796Z INFO mlxcel::server::startup: Runtime device: NVIDIA GPU (CUDA) +2026-10-11T06:15:22.049799Z INFO mlxcel::server::startup: CUDA graph-cache LRU capacity: MLX_CUDA_GRAPH_CACHE_SIZE=2000 (mlxcel raises MLX's default of 400 to 2000 so long-lived, shape-diverse decode does not hit the cache-thrashing abort from issue #818, unless an operator override is set) +2026-10-11T06:15:22.049801Z INFO mlxcel::server::startup: Wired memory limit: 121.7 GB +2026-10-11T06:15:22.540377Z INFO mlxcel::server::startup: DRY sequence breakers active (b10621 semantics: breaker token data derived from the vocabulary per request) breakers=["\n", ":", "\"", "*"] +2026-10-11T06:15:22.552190Z INFO mlxcel::server::startup: Applied model-aware prompt-cache snapshot capacity default previous_snapshot_capacity_bytes=536870912 recommended_snapshot_capacity_bytes=9663676416 representative_snapshot_bytes=1610612736 representative_tokens=8192 target_entries=6 kv_bytes_at_representative_tokens=1610612736 fixed_state_bytes=0 available_ceiling_bytes=12186870490 architecture=standard attention (48 layers, full context) +2026-10-11T06:15:22.552877Z INFO mlxcel::server::startup: Prompt-prefix cache store enabled (+ APC, snapshots) capacity_bytes=2147483648 max_entries=1024 ttl_seconds=3600 snapshot_capacity_bytes=9663676416 snapshot_max_entries=4096 snapshot_ttl_seconds=7200 min_prefix_tokens=32 apc_enabled=true apc_block_size=16 apc_hash=sha256 +2026-10-11T06:15:23.002390Z INFO mlxcel::server::startup: Warming up model... +2026-10-11T06:15:23.002396Z INFO mlxcel::server::model_provider::model_worker: Model worker thread starting, loading model... +2026-10-11T06:15:23.518727Z INFO mlxcel::server::model_provider::model_worker: Model llama-4-scout-17b-16e-instruct-4bit loaded in 0.516s (resident after load: 0.00 GB) worker_model_id=llama-4-scout-17b-16e-instruct-4bit load_seconds=0.516285056 active_bytes=229376 peak_bytes=229376 cache_bytes=0 limit_bytes=124128081510 +2026-10-11T06:15:23.518794Z INFO mlxcel::server::model_provider::model_worker: EOS tokens from config: [200001, 200007, 200008] +2026-10-11T06:15:23.518810Z INFO mlxcel::server::model_provider::model_worker: Starting BatchScheduler (max_batch_size=8, max_queue_depth=32, prefill_chunk_size=2048, max_batch_prefill=4, decode_storage=paged) +2026-10-11T06:15:23.519642Z INFO mlxcel::server::model_provider::model_worker: Paged KV block budget: 317207 blocks (48 layers, 32-token blocks) +2026-10-11T06:15:23.519684Z INFO mlxcel::server::model_provider::model_worker: Paged KV slab size: 2048 blocks per layer (fused decode serves a layer only while its rows fit one slab) +2026-10-11T06:15:23.519711Z INFO mlxcel::server::batch::scheduler::paged_layout: resolved KV cache mode applied to model caches kv_cache_mode_effective=fp16 kv_cache_mode_applied_layers=48 kv_cache_mode_total_layers=48 +2026-10-11T06:15:23.521075Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel_core::sampling_dispatch: sampling dispatch: argmax: greedy path (temperature 0, top_k 1); no sampling kernel involved +2026-10-11T06:15:56.796550Z INFO prefill{seq_id=seq-0 prompt_len=2 cached=0 start=0}: mlxcel::server::batch::scheduler::prefill: prompt-cache: request completed during prefill: cached=0/2 prompt tokens, total 33276ms prompt_tokens=2 cached_tokens=0 generation_time_ms=33276 +2026-10-11T06:15:56.797489Z INFO mlxcel::server::startup: Warmup complete +2026-10-11T06:15:56.797624Z INFO mlxcel::server::startup: model_type=Llama4VLM: no native video path; video_url content blocks will be served as ordered sampled frames +2026-10-11T06:15:56.798595Z WARN mlxcel::server::startup: The loaded model accepts video input, but `ffmpeg` and `ffprobe` are not both on PATH, so every video request will be refused. The check is cached for the life of the process: restart the server after installing ffmpeg. +2026-10-11T06:15:56.799461Z INFO mlxcel::server::startup: Starting mlxcel server on http://127.0.0.1:18080 +2026-10-11T06:15:56.799466Z INFO mlxcel::server::startup: Detected 1 GPU(s) +2026-10-11T06:15:56.799469Z INFO mlxcel::server::startup: CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +2026-10-11T06:15:56.799471Z INFO mlxcel::server::startup: Endpoints: +2026-10-11T06:15:56.799472Z INFO mlxcel::server::startup: POST /v1/chat/completions - OpenAI chat completions +2026-10-11T06:15:56.799474Z INFO mlxcel::server::startup: POST /v1/completions - OpenAI text completions +2026-10-11T06:15:56.799475Z INFO mlxcel::server::startup: GET /v1/models - List models +2026-10-11T06:15:56.799476Z INFO mlxcel::server::startup: POST /completion - llama-server native completion +2026-10-11T06:15:56.799478Z INFO mlxcel::server::startup: POST /tokenize - Tokenize text +2026-10-11T06:15:56.799479Z INFO mlxcel::server::startup: POST /detokenize - Detokenize tokens +2026-10-11T06:15:56.799481Z INFO mlxcel::server::startup: GET /props - Server properties +2026-10-11T06:15:56.799482Z INFO mlxcel::server::startup: GET /slots - Slot status +2026-10-11T06:15:56.799484Z INFO mlxcel::server::startup: GET /health - Health check +2026-10-11T06:15:57.188048Z INFO mlxcel::server::chat_request: prompt cache on + preserve_thinking unset: defaulting preserve_thinking=true for prefix stability (override via chat_template_kwargs.preserve_thinking=false) session=__mlxcel_anon__ +2026-10-11T06:16:08.078909Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=0/442 prompt tokens, total 10889ms prompt_tokens=442 cached_tokens=0 generation_time_ms=10889 +2026-10-11T06:16:24.939009Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16680ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16680 +2026-10-11T06:16:24.939689Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16681ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16681 +2026-10-11T06:16:24.940216Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16681ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16681 +2026-10-11T06:16:24.940880Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 16681ms prompt_tokens=442 cached_tokens=441 generation_time_ms=16681 +2026-10-11T06:17:21.992515Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56868ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56868 +2026-10-11T06:17:21.993460Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56868ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56868 +2026-10-11T06:17:21.994314Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56868ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56868 +2026-10-11T06:17:21.995025Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56870ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56870 +2026-10-11T06:17:21.995690Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56870ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56870 +2026-10-11T06:17:21.996379Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56870ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56870 +2026-10-11T06:17:21.997111Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56872ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56872 +2026-10-11T06:17:21.997679Z INFO mlxcel::server::batch::scheduler::decode_tick: prompt-cache: request completed: cached=441/442 prompt tokens, total 56871ms prompt_tokens=442 cached_tokens=441 generation_time_ms=56871 diff --git a/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/README.md b/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/README.md new file mode 100644 index 000000000..db941ad2e --- /dev/null +++ b/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/README.md @@ -0,0 +1,29 @@ +# Raw data for CUDA bf16 SSM parity on GB10 (2026-10-11) + +This directory preserves the numeric diagnosis and real-checkpoint parity evidence for issue #2230. + +## Stage deviations + +`stage-deviations.log` contains the exact command records and stdout from the original seed-2069 probe. They were recovered from the original execution record and were not recreated for this report. The extraction places the fixed run before the failing baseline run; the timestamps and command tags identify their actual order and status. + +The baseline compiled `dt'` had bf16 dtype and diverged from the stable float32 graph at normalized RMS/max `1.9571408311071774e-3 / 5.875451501701449e-3`. Its decay and final state deviations were `3.008783834355594e-3 / 1.0758810176786593e-2` and `2.5324028008192334e-3 / 4.127457397370858e-2`. After explicitly casting `dt_bias` to float32 beside `dt`, the three deviations fell to `2.890811209876877e-8 / 2.0028127562385632e-7`, `6.809246821557169e-10 / 4.717584582485562e-9`, and `2.838613609145525e-8 / 1.7010096528383182e-6` without changing the tolerance. + +## Granite checkpoint parity + +`granite-parity/` records a deterministic greedy comparison on `models/mlx/granite-4.0-h-tiny-4bit`: + +- `run.sh`: exact kernel-versus-graph command and comparison driver. +- `provenance.txt`: source commit, release binary SHA-256, and shell-escaped commands. +- `kernel.stdout`, `kernel.stderr`: fused CUDA SSM arm output. +- `graph.stdout`, `graph.stderr`: graph arm output with `MLXCEL_SSM_CUDA_KERNEL=0`. +- `comparison.json`: 16 prompt IDs, 64 generated IDs, both complete ID arrays, equality result, and generated-token SHA-256. + +Both arms used prompt `Explain the Mamba state update.`, `-n 64`, `--temp 0`, `MLXCEL_SDPA_DETERMINISTIC=1`, and token-ID printing. All 64 generated IDs were identical. Their SHA-256 is `9df6f4fc2abbfcb121fa5dca3ad1e140e8db5b67d28118a905b93022f8905829`. + +The recorded release binary SHA-256 is `0cc49217021dab12b5add25dc2bb8de7e90c405c0b406607b244a8060d3ed800`. `SHA256SUMS` covers every other file in this directory. + +Validate the preserved files from this directory with: + +```bash +sha256sum --check SHA256SUMS +``` diff --git a/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/SHA256SUMS b/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/SHA256SUMS new file mode 100644 index 000000000..9d7fd105c --- /dev/null +++ b/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/SHA256SUMS @@ -0,0 +1,9 @@ +14b73bcdf4de9f1361bb44249e184d1a032216908052a28a92bcdaa418a0f4ae README.md +aabe38700bc3adfd7bb832d211b0eac5bf7342f4e476490ab6d551cc697c4b76 granite-parity/comparison.json +726bf4f6ad48e396b97c8e109f34f84bf103c44aa8a15ce7d790e22e75eefd2a granite-parity/graph.stderr +09e8c5ea915df19a7eebc1ec4953e94a93367730ade340146cdb5b95b0622ff9 granite-parity/graph.stdout +5a9227f2f7adf68702ab8a99fb26e5402d237875aa896e11f921aaffc55c867f granite-parity/kernel.stderr +2ae631946b4c259c11c4b0240d9330a80721028e16e1ad2665085aa29f8a3eff granite-parity/kernel.stdout +4f7f80359779669005822023916527675d7db5d713192847fc36c26716cef759 granite-parity/provenance.txt +33432ac54861427d55ac9e46303800065c1a9a9eab30d2b2108655596c3c22ae granite-parity/run.sh +6ce458974415d0a968acd85f0c696ebb7a8051b36860b41dd554880d2ac84e88 stage-deviations.log diff --git a/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/comparison.json b/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/comparison.json new file mode 100644 index 000000000..0ba82715c --- /dev/null +++ b/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/comparison.json @@ -0,0 +1,139 @@ +{ + "prompt_ids": 16, + "generated_ids": 64, + "token_ids_identical": true, + "first_divergence_zero_based": null, + "kernel_token_ids": [ + 791, + 386, + 43008, + 1614, + 2713, + 374, + 264, + 15105, + 1511, + 304, + 279, + 386, + 43008, + 30828, + 4009, + 18112, + 11, + 902, + 374, + 264, + 955, + 315, + 65174, + 30828, + 4009, + 320, + 49, + 9944, + 8, + 6319, + 369, + 11297, + 323, + 7524, + 8668, + 34579, + 13, + 578, + 386, + 43008, + 18112, + 22262, + 311, + 2686, + 1063, + 315, + 279, + 9669, + 315, + 8776, + 432, + 9944, + 82, + 11, + 1778, + 439, + 279, + 17250, + 304, + 40880, + 1317, + 31608, + 20113, + 323 + ], + "graph_token_ids": [ + 791, + 386, + 43008, + 1614, + 2713, + 374, + 264, + 15105, + 1511, + 304, + 279, + 386, + 43008, + 30828, + 4009, + 18112, + 11, + 902, + 374, + 264, + 955, + 315, + 65174, + 30828, + 4009, + 320, + 49, + 9944, + 8, + 6319, + 369, + 11297, + 323, + 7524, + 8668, + 34579, + 13, + 578, + 386, + 43008, + 18112, + 22262, + 311, + 2686, + 1063, + 315, + 279, + 9669, + 315, + 8776, + 432, + 9944, + 82, + 11, + 1778, + 439, + 279, + 17250, + 304, + 40880, + 1317, + 31608, + 20113, + 323 + ], + "kernel_token_ids_sha256": "9df6f4fc2abbfcb121fa5dca3ad1e140e8db5b67d28118a905b93022f8905829" +} diff --git a/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/graph.stderr b/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/graph.stderr new file mode 100644 index 000000000..019093ed4 --- /dev/null +++ b/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/graph.stderr @@ -0,0 +1,2 @@ +[prompt ids (16): 100264 882 100265 849 21435 279 386 43008 1614 2713 13 100257 198 100264 78191 100265] +[token ids (64): 791 386 43008 1614 2713 374 264 15105 1511 304 279 386 43008 30828 4009 18112 11 902 374 264 955 315 65174 30828 4009 320 49 9944 8 6319 369 11297 323 7524 8668 34579 13 578 386 43008 18112 22262 311 2686 1063 315 279 9669 315 8776 432 9944 82 11 1778 439 279 17250 304 40880 1317 31608 20113 323] diff --git a/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/graph.stdout b/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/graph.stdout new file mode 100644 index 000000000..1b45f134e --- /dev/null +++ b/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/graph.stdout @@ -0,0 +1,12 @@ +Runtime device: NVIDIA GPU (CUDA) +Wired memory limit: 121.7 GB +KV cache mode: fp16 (applied to 40 of 40 layers) +Loading model from "/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/models/mlx/granite-4.0-h-tiny-4bit"... +Detected 1 GPU(s). +CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +GPU: NVIDIA GB10 (Nvidia), 121.69 GiB device memory. +Model loaded in 0.192s (resident: 0.00 GB, peak: 0.00 GB). +Generating... +Explain the Mamba state update.The Mamba state update is a technique used in the Mamba neural network architecture, which is a type of recurrent neural network (RNN) designed for efficient and effective sequence modeling. The Mamba architecture aims to address some of the limitations of traditional RNNs, such as the difficulty in capturing long-range dependencies and + +[Generated 64 tokens in 2.37s = 27.04 tok/s] diff --git a/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/kernel.stderr b/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/kernel.stderr new file mode 100644 index 000000000..6b051e60e --- /dev/null +++ b/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/kernel.stderr @@ -0,0 +1,3 @@ +Compiling CUDA kernels (first run on this host; cached for later runs)... +[prompt ids (16): 100264 882 100265 849 21435 279 386 43008 1614 2713 13 100257 198 100264 78191 100265] +[token ids (64): 791 386 43008 1614 2713 374 264 15105 1511 304 279 386 43008 30828 4009 18112 11 902 374 264 955 315 65174 30828 4009 320 49 9944 8 6319 369 11297 323 7524 8668 34579 13 578 386 43008 18112 22262 311 2686 1063 315 279 9669 315 8776 432 9944 82 11 1778 439 279 17250 304 40880 1317 31608 20113 323] diff --git a/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/kernel.stdout b/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/kernel.stdout new file mode 100644 index 000000000..43bed7c1b --- /dev/null +++ b/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/kernel.stdout @@ -0,0 +1,12 @@ +Runtime device: NVIDIA GPU (CUDA) +Wired memory limit: 121.7 GB +KV cache mode: fp16 (applied to 40 of 40 layers) +Loading model from "/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/models/mlx/granite-4.0-h-tiny-4bit"... +Detected 1 GPU(s). +CUDA compute capability 12.1 (sm_121); compiled for [121] (cubin) +GPU: NVIDIA GB10 (Nvidia), 121.69 GiB device memory. +Model loaded in 0.178s (resident: 0.00 GB, peak: 0.00 GB). +Generating... +Explain the Mamba state update.The Mamba state update is a technique used in the Mamba neural network architecture, which is a type of recurrent neural network (RNN) designed for efficient and effective sequence modeling. The Mamba architecture aims to address some of the limitations of traditional RNNs, such as the difficulty in capturing long-range dependencies and + +[Generated 64 tokens in 6.03s = 10.61 tok/s] diff --git a/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/provenance.txt b/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/provenance.txt new file mode 100644 index 000000000..84e4b6f46 --- /dev/null +++ b/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/provenance.txt @@ -0,0 +1,5 @@ +run_dir=/tmp/mlxcel-auto-plan/2230-granite-parity.HxKrMPdk +git_commit=fb2cb7832028c7425dc347c23cc4fdaeb1d64d7c +binary_sha256=0cc49217021dab12b5add25dc2bb8de7e90c405c0b406607b244a8060d3ed800 +kernel_command=env -u MLXCEL_SSM_KERNEL -u MLXCEL_SSM_CUDA_KERNEL MLXCEL_SDPA_DETERMINISTIC=1 MLXCEL_PRINT_TOKEN_IDS=1 /home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/target/release/mlxcel generate -m /home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/models/mlx/granite-4.0-h-tiny-4bit -p Explain\ the\ Mamba\ state\ update. -n 64 --temp 0 +graph_command=env -u MLXCEL_SSM_KERNEL MLXCEL_SDPA_DETERMINISTIC=1 MLXCEL_PRINT_TOKEN_IDS=1 MLXCEL_SSM_CUDA_KERNEL=0 /home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/target/release/mlxcel generate -m /home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/models/mlx/granite-4.0-h-tiny-4bit -p Explain\ the\ Mamba\ state\ update. -n 64 --temp 0 diff --git a/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/run.sh b/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/run.sh new file mode 100755 index 000000000..1e919ac53 --- /dev/null +++ b/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/granite-parity/run.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT=/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230 +BIN="$ROOT/target/release/mlxcel" +MODEL="$ROOT/models/mlx/granite-4.0-h-tiny-4bit" +PROMPT='Explain the Mamba state update.' +OUT_ROOT=/tmp/mlxcel-auto-plan + +if [[ ${ISSUE2230_GRANITE_LOCKED:-0} != 1 ]]; then + command -v gpu-lock >/dev/null + exec gpu-lock run --tag issue2230-granite-parity -- \ + env ISSUE2230_GRANITE_LOCKED=1 "$0" "$@" +fi + +[[ -x "$BIN" ]] || { + echo "missing release binary: $BIN" >&2 + exit 1 +} +[[ -f "$MODEL/config.json" ]] || { + echo "missing Granite checkpoint: $MODEL" >&2 + exit 1 +} + +mkdir -p "$OUT_ROOT" +RUN_DIR=$(mktemp -d "$OUT_ROOT/2230-granite-parity.XXXXXXXX") +COMMON=( + generate + -m "$MODEL" + -p "$PROMPT" + -n 64 + --temp 0 +) + +{ + echo "run_dir=$RUN_DIR" + echo "git_commit=$(git -C "$ROOT" rev-parse HEAD)" + echo "binary_sha256=$(sha256sum "$BIN" | awk '{print $1}')" + printf 'kernel_command=' + printf '%q ' env -u MLXCEL_SSM_KERNEL -u MLXCEL_SSM_CUDA_KERNEL \ + MLXCEL_SDPA_DETERMINISTIC=1 MLXCEL_PRINT_TOKEN_IDS=1 \ + "$BIN" "${COMMON[@]}" + printf '\n' + printf 'graph_command=' + printf '%q ' env -u MLXCEL_SSM_KERNEL \ + MLXCEL_SDPA_DETERMINISTIC=1 MLXCEL_PRINT_TOKEN_IDS=1 \ + MLXCEL_SSM_CUDA_KERNEL=0 "$BIN" "${COMMON[@]}" + printf '\n' +} | tee "$RUN_DIR/provenance.txt" + +env -u MLXCEL_SSM_KERNEL -u MLXCEL_SSM_CUDA_KERNEL \ + MLXCEL_SDPA_DETERMINISTIC=1 \ + MLXCEL_PRINT_TOKEN_IDS=1 \ + "$BIN" "${COMMON[@]}" \ + >"$RUN_DIR/kernel.stdout" \ + 2>"$RUN_DIR/kernel.stderr" + +env -u MLXCEL_SSM_KERNEL \ + MLXCEL_SDPA_DETERMINISTIC=1 \ + MLXCEL_PRINT_TOKEN_IDS=1 \ + MLXCEL_SSM_CUDA_KERNEL=0 \ + "$BIN" "${COMMON[@]}" \ + >"$RUN_DIR/graph.stdout" \ + 2>"$RUN_DIR/graph.stderr" + +python3 - "$RUN_DIR" <<'PY' +import hashlib +import json +import pathlib +import re +import sys + +run_dir = pathlib.Path(sys.argv[1]) +line_pattern = re.compile(r"\[(prompt|token) ids \((\d+)\): ([0-9 ]*)\]") + + +def read_ids(arm: str) -> dict[str, list[int]]: + text = (run_dir / f"{arm}.stderr").read_text() + found: dict[str, list[int]] = {} + for kind, declared_text, payload in line_pattern.findall(text): + ids = [int(value) for value in payload.split()] if payload.strip() else [] + declared = int(declared_text) + if declared != len(ids): + raise SystemExit( + f"{arm}: {kind} id line declares {declared}, contains {len(ids)}" + ) + if kind in found: + raise SystemExit(f"{arm}: duplicate {kind} id line") + found[kind] = ids + if set(found) != {"prompt", "token"}: + raise SystemExit( + f"{arm}: expected prompt and token id lines, found {sorted(found)}" + ) + if len(found["token"]) != 64: + raise SystemExit( + f"{arm}: expected 64 generated token ids, found {len(found['token'])}" + ) + return found + + +kernel = read_ids("kernel") +graph = read_ids("graph") +if kernel["prompt"] != graph["prompt"]: + raise SystemExit("kernel and graph arms tokenized the prompt differently") + +first_divergence = next( + ( + index + for index, (kernel_id, graph_id) in enumerate( + zip(kernel["token"], graph["token"], strict=True) + ) + if kernel_id != graph_id + ), + None, +) +payload = " ".join(map(str, kernel["token"])).encode() +comparison = { + "prompt_ids": len(kernel["prompt"]), + "generated_ids": len(kernel["token"]), + "token_ids_identical": first_divergence is None, + "first_divergence_zero_based": first_divergence, + "kernel_token_ids": kernel["token"], + "graph_token_ids": graph["token"], + "kernel_token_ids_sha256": hashlib.sha256(payload).hexdigest(), +} +(run_dir / "comparison.json").write_text(json.dumps(comparison, indent=2) + "\n") +print(json.dumps(comparison, indent=2)) +if first_divergence is not None: + raise SystemExit( + "Granite greedy kernel/graph outputs diverged at generated token " + f"{first_divergence + 1}" + ) +PY + +echo "Granite kernel and graph paths produced the same 64 greedy token ids." +echo "Artifacts: $RUN_DIR" diff --git a/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/stage-deviations.log b/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/stage-deviations.log new file mode 100644 index 000000000..c799d6f4a --- /dev/null +++ b/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/stage-deviations.log @@ -0,0 +1,39 @@ +timestamp=2026-10-11T02:59:19.975Z +command=/bin/zsh -lc gpu-lock run --tag issue2230-ssm-fixed-decay -- target/test-fast/deps/mlxcel_core-5feb86ac05c84f53 ssm_update_parity_tests::ssm_update_kernel_matches_graph_step_bf16 --exact --test-threads=1 --nocapture +status=completed +exit_code=0 +--- output --- + +running 1 test +test ssm_update_parity_tests::ssm_update_kernel_matches_graph_step_bf16 ... Compiling CUDA kernels (first run on this host; cached for later runs)... +SSM stage probe seed 2069: compiled dt dtype 10; dt compiled/stable (2.890811209876877e-8, 2.0028127562385632e-7); dt compiled/bf16-rounded (0.0015696586690978889, 0.005879791600633243); dt naive/stable (2.890811209876877e-8, 2.0028127562385632e-7); decay compiled/stable (6.809246821557169e-10, 4.717584582485562e-9); decay naive/stable (6.809246821557169e-10, 4.717584582485562e-9); decay rounded-A/stable (0.0008696147800822081, 0.0023052609154684007); state kernel/naive (2.34598996877995e-8, 8.505048240025111e-7); state kernel/rounded-A (0.0007675677693150441, 0.00796170226000352); state kernel/stable (2.838613609145525e-8, 1.7010096528383182e-6) +ok + +test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 2052 filtered out; finished in 1.91s + +--- end output --- + +timestamp=2026-10-11T02:59:52.462Z +command=/bin/zsh -lc gpu-lock run --tag issue2230-ssm-baseline-decay -- target/test-fast/deps/mlxcel_core-5feb86ac05c84f53 ssm_update_parity_tests::ssm_update_kernel_matches_graph_step_bf16 --exact --test-threads=1 --nocapture +status=failed +exit_code=101 +--- output --- + +running 1 test +test ssm_update_parity_tests::ssm_update_kernel_matches_graph_step_bf16 ... Compiling CUDA kernels (first run on this host; cached for later runs)... +SSM stage probe seed 2069: compiled dt dtype 12; dt compiled/stable (0.0019571408311071774, 0.005875451501701449); dt compiled/bf16-rounded (0.0017737195567379125, 0.0065676646839906636); dt naive/stable (2.890811209876877e-8, 2.0028127562385632e-7); decay compiled/stable (0.003008783834355594, 0.010758810176786593); decay naive/stable (6.809246821557169e-10, 4.717584582485562e-9); decay rounded-A/stable (0.0008696147800822081, 0.0023052609154684007); state kernel/naive (0.0025324044936919097, 0.04127457385642986); state kernel/rounded-A (0.002590295934443625, 0.040084687630816164); state kernel/stable (0.0025324028008192334, 0.04127457397370858) + +thread 'ssm_update_parity_tests::ssm_update_kernel_matches_graph_step_bf16' (4073340) panicked at src/lib/mlxcel-core/src/ssm_update_parity_tests.rs:131:5: +granite-4.0-h-tiny act dtype 12 A_log dtype 12 on Cuda: new state: normalized rms 2.532e-3 (tol 1.0e-5), normalized max 4.127e-2 (tol 1.0e-4) +note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace +FAILED + +failures: + +failures: + ssm_update_parity_tests::ssm_update_kernel_matches_graph_step_bf16 + +test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 2052 filtered out; finished in 1.13s + +--- end output --- + diff --git a/docs/benchmark_results/model-owned-storage-gb10-2026-10-11.md b/docs/benchmark_results/model-owned-storage-gb10-2026-10-11.md new file mode 100644 index 000000000..730f6d1ef --- /dev/null +++ b/docs/benchmark_results/model-owned-storage-gb10-2026-10-11.md @@ -0,0 +1,75 @@ +# Model-owned decode storage on GB10 (2026-10-11) + +Issue #2230 required a paired/null serving sweep before deciding whether Gemma 3 and Llama 4 should continue to advertise paged decode storage. Both flags remain `true`. Across the six prompt/concurrency cells for each family, neither dense nor paged storage cleared the repeated-paged null spread under the preregistered rule. + +This decision concerns storage allocation and the per-step model-owned KV mirror. These model families continue to compute attention from their dense rows; selecting paged storage does not imply that their attention reads the paged pool. + +## Environment and provenance + +- Host: `spark-102`, NVIDIA GB10, driver `580.178.04`, PCI `0000000F:01:00.0` +- Run interval: `2026-10-11T04:57:35Z` through `2026-10-11T07:12:28Z` +- Clean measurement checkout: `35da2dbe32c731ff6645b2b32a2117af2965f332` +- Release binary production source: `2cb92404b153897440c4906a5548dcfb4889a9ad` (post-run derived verification described below) +- Release binary SHA-256: `0cc49217021dab12b5add25dc2bb8de7e90c405c0b406607b244a8060d3ed800` +- Release binary size and mtime: `108106312` bytes, `2026-10-11T03:25:49Z` +- Configured and built MLX commit: `81ba1c6a0e50a9268b931579c2d4f1158b9aab5a` +- Models: `gemma-3-1b-it-4bit` and `llama-4-scout-17b-16e-instruct-4bit` +- Outer lock: `gpu-lock run --tag bench-2230-storage -- .../run.sh` + +The committed [raw evidence](data/model-owned-storage-gb10-2026-10-11/README.md) includes the exact contemporaneous environment, measured script hashes, schedule, host-gate snapshots, server and client output, analyzer output, completion marker, checksums, and a separate derived binary-source proof. + +`environment.txt` uses `commit` for the clean checkout that ran the sweep. The release binary does not embed a source revision, so its association with `2cb92404` was reconstructed after the run from the preserved release-build session, the artifact hash and mtime, and Git history. Commit `2cb92404` is an ancestor of `35da2dbe`, and the complete intervening diff contains only this run's `analyze.py`, `gate.py`, `run.sh`, and `test_protocol.py`. No Rust, C++, model, or build-definition path changed. The original environment and measured script bytes and hashes remain unchanged. [The companion provenance record](data/model-owned-storage-gb10-2026-10-11/binary-source-provenance.json) records the commands, exact paths, release-log hash, and limitation. + +## Method + +Each arm started a fresh `mlxcel serve --parallel 8` process on port 18080 with an explicit `--decode-storage-backend paged|dense`. The client requested 128 generated tokens for concurrency 1, 4, and 8, first with a 512-token prompt and then with a 4096-token prompt. Five rounds were run for each model and prompt length. + +Every round contained `paged-a`, `dense`, and `paged-b`. Their order rotated across rounds to distribute position effects. This produced 60 server arms and 180 successful client cells: + +```text +2 models x 2 prompt lengths x 5 rounds x 3 arms = 60 arms +60 arms x 3 concurrency levels = 180 cells +``` + +The driver ran inside one GPU lock. Before the run and before every arm, its fail-closed gate required port 18080 to be free, a sustained quiet-host interval, no active CI/build/foreign-model processes, acceptable memory pressure, and clean NVIDIA driver state. It repeated the checks after the quiet interval and recorded only accepted snapshots. The analyzer additionally required the completion marker, a clean source commit and artifact provenance, matching configured and built MLX commits, exact script hashes, the exact ordered 60-arm schedule, 61 ordered quiet-gate records, 60 nonempty backend-marked server logs, 60 client logs, and the exact 180 successful measurement keys. + +`paged-a` is the paired baseline. For every round, the dense paired delta is `dense / paged-a - 1`, while the null delta is `paged-b / paged-a - 1`. A cell is a clear dense win only when the median paired delta is greater than `max(0, null_high)`. A cell is a clear paged win only when the median paired delta is less than `min(0, null_low)`. Per family, dense storage would replace paged storage only if at least one cell were a clear dense win and no cell were a clear paged win. + +## Results + +Rates are aggregate generated tokens per second. Parenthesized ranges cover the five round-level paired deltas; the null spread is the range of the five repeated-paged deltas. + +| Model | Prompt | C | paged-a median | dense median | paged-b median | dense paired delta | null spread | Cell | +|---|---:|---:|---:|---:|---:|---:|---:|---| +| gemma3 | 512 | 1 | 154.4 | 154.0 | 154.0 | +0.52% (-2.76..+3.63) | -2.37..+3.84% | unresolved | +| gemma3 | 512 | 4 | 361.0 | 363.6 | 360.6 | +1.29% (-2.11..+2.49) | -1.35..+3.52% | unresolved | +| gemma3 | 512 | 8 | 194.9 | 197.2 | 194.4 | +1.16% (+0.25..+6.37) | -0.88..+3.80% | unresolved | +| gemma3 | 4096 | 1 | 114.6 | 114.4 | 113.0 | -0.18% (-1.82..+0.26) | -2.33..+1.31% | unresolved | +| gemma3 | 4096 | 4 | 219.4 | 216.6 | 218.6 | -0.60% (-2.74..+0.54) | -3.45..+0.50% | unresolved | +| gemma3 | 4096 | 8 | 184.4 | 185.8 | 184.0 | +0.49% (-0.53..+1.98) | -3.13..+1.91% | unresolved | +| llama4 | 512 | 1 | 11.7 | 11.8 | 11.8 | +0.85% (+0.00..+83.08) | -0.85..+81.54% | unresolved | +| llama4 | 512 | 4 | 31.0 | 30.8 | 30.7 | +0.65% (-3.53..+2.61) | -0.97..+3.53% | unresolved | +| llama4 | 512 | 8 | 17.8 | 17.9 | 17.7 | +0.00% (-1.69..+13.29) | -1.13..+10.76% | unresolved | +| llama4 | 4096 | 1 | 8.0 | 8.0 | 8.0 | +1.25% (+0.00..+1.27) | +0.00..+2.53% | unresolved | +| llama4 | 4096 | 4 | 22.9 | 23.1 | 23.0 | +0.44% (-0.43..+2.25) | -2.99..+4.05% | unresolved | +| llama4 | 4096 | 8 | 14.7 | 14.7 | 14.5 | +0.68% (-0.68..+3.55) | -2.03..+2.84% | unresolved | + +No cell met either clear-win condition. `Gemma3Wrapper::supports_paged_decode_backend()` and `Llama4Wrapper::supports_paged_decode_backend()` therefore remain `true`. `auto` on a multi-slot server and an explicit `paged` request continue to select paged storage for both families. + +## Observed first-arm anomaly + +The first Llama 4 round at a 512-token prompt contained a large anomaly in its first arm, `paged-a`, at concurrency 1. That arm recorded 6.5 aggregate tok/s and 12690.4 ms mean TTFT. The following dense and `paged-b` arms recorded 11.9 and 11.8 aggregate tok/s with 4072.6 and 4128.2 ms TTFT. Rounds 2 through 5 recorded approximately 11.7 to 11.9 aggregate tok/s across all three arms at concurrency 1. + +Consequently, round 1 contributes a +83.08% dense paired delta and a +81.54% repeated-paged null delta. All records are retained exactly as measured. The host gate accepted the arm, and the evidence does not establish the anomaly's cause. A cold or warm-up effect is one possible hypothesis, but it was not measured and is not asserted here. The preregistered paired/null rule absorbs the observed variation: the five-round paired median is only +0.85%, while the null upper bound is +81.54%, so the cell remains unresolved. The run was not repeated or filtered after observing the result. + +## Reproduction + +The driver contains absolute paths because those paths are part of the run provenance. On a matching checkout, model layout, release artifact, and MLX build tree, run it under the GPU lock: + +```bash +gpu-lock run --tag bench-2230-storage -- bash docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/run.sh +python3 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/analyze.py \ + --data-dir docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11 +``` + +The committed `complete` marker prevents accidental overwrite. A deliberate rerun should use a new dated directory and preserve this run unchanged. diff --git a/src/models/gemma3.rs b/src/models/gemma3.rs index c05bd625f..578029353 100644 --- a/src/models/gemma3.rs +++ b/src/models/gemma3.rs @@ -1617,6 +1617,10 @@ impl mlxcel_core::generate::LanguageModel for Gemma3Wrapper { })? } + /// Retained after the 2026-10-11 GB10 paired/null storage sweep in + /// `docs/benchmark_results/model-owned-storage-gb10-2026-10-11.md`. + /// Across 512/4096-token prompts at concurrency 1/4/8, neither backend + /// cleared the paged-repeat null spread in any Gemma 3 cell. fn supports_paged_decode_backend(&self) -> bool { true } diff --git a/src/models/llama4.rs b/src/models/llama4.rs index 3b5269e61..74d6eaa03 100644 --- a/src/models/llama4.rs +++ b/src/models/llama4.rs @@ -1799,6 +1799,10 @@ impl LanguageModel for Llama4Wrapper { false } + /// Retained after the 2026-10-11 GB10 paired/null storage sweep in + /// `docs/benchmark_results/model-owned-storage-gb10-2026-10-11.md`. + /// Across 512/4096-token prompts at concurrency 1/4/8, neither backend + /// cleared the paged-repeat null spread in any Llama 4 cell. fn supports_paged_decode_backend(&self) -> bool { true } From 479b3585efa5d73c29b2e6504cfc81f5eb53dfdb Mon Sep 17 00:00:00 2001 From: Jeongkyu Shin Date: Sun, 11 Oct 2026 16:22:51 +0900 Subject: [PATCH 5/7] docs: preserve release build provenance Commit the release-build transcript referenced by the storage sweep's derived binary-source proof. Clarify that the transcript proves the successful 19m43s build but does not print HEAD; the production-source association remains reconstructed from session context, artifact identity, ancestry, and the protocol-only intervening diff. Validation: the release-log SHA-256 matches the provenance record and the refreshed evidence manifest verifies. Refs #2230 --- .../README.md | 3 +- .../SHA256SUMS | 5 +- .../binary-source-provenance.json | 6 +- .../release-build.log | 128 ++++++++++++++++++ .../model-owned-storage-gb10-2026-10-11.md | 2 +- 5 files changed, 137 insertions(+), 7 deletions(-) create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/release-build.log diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/README.md b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/README.md index 4827e965f..4e8eab5a6 100644 --- a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/README.md +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/README.md @@ -9,7 +9,8 @@ This directory contains the complete accepted run for the Gemma 3 and Llama 4 mo - `analyze.py`: protocol and provenance validator plus paired/null decision analyzer. - `test_protocol.py`: lightweight synthetic fixtures for the valid protocol and rejected missing, dirty, reordered, nonmonotonic, incomplete, or backend-substituted evidence. - `environment.txt`: contemporaneous start/end times, clean benchmark-checkout commit, release artifact identity, configured and built MLX commits, host/GPU identity, host-gate identity, and the measured gate/analyzer/driver hashes. Its `commit` key identifies the checkout used to run the sweep, not an embedded binary-source revision. -- `binary-source-provenance.json`: post-run derived verification that identifies the release binary's production source commit, proves it is an ancestor of the benchmark checkout, and records the exact four protocol-only paths in the intervening diff. It also records the preserved release-log reference and hash. This companion does not rewrite or claim to be part of the contemporaneous environment record. +- `binary-source-provenance.json`: post-run derived verification that identifies the release binary's production source commit, proves it is an ancestor of the benchmark checkout, and records the exact four protocol-only paths in the intervening diff. This companion does not rewrite or claim to be part of the contemporaneous environment record. +- `release-build.log`: preserved release-build transcript. It records a successful 19m43s build and has the hash named by the derived provenance record. The transcript does not print HEAD, so it supports artifact timing and build success while the source association remains an explicitly derived result. - `schedule.tsv`: the exact ordered 60-arm schedule. The header plus 60 arms gives 61 lines. - `hostgate.jsonl`: one accepted preflight snapshot followed by one accepted snapshot for each arm, in schedule order (61 records total). - `server-*.log`: 60 nonempty server logs. Each begins with the requested dense or paged backend marker used by the analyzer. diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/SHA256SUMS b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/SHA256SUMS index af20a4d86..a15edd26e 100644 --- a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/SHA256SUMS +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/SHA256SUMS @@ -1,7 +1,7 @@ -53752a55caa2f4fa517081c1695e8ca56c7fe6a31e8d898e80fe1f9ffb3f437c README.md +a7375a6a8077e73cc4afca3aa8125072eeffe4125821410f7e83ae4ffccb18cb README.md 6e27465a51285608dbbd38798864556ee8b8cca4bf2c422fb965e45dfd60bce7 analysis.txt f17233d3e5b5339424c050404ea81a171ad43b0abd74ebb4d4ddc681f894f3f8 analyze.py -33d55685f078a9b3f902ff8144b0aad525081d7199d94bcf7dcc77f22a0f9011 binary-source-provenance.json +a18eb78ca4520b4098b14f85f899cc6d16df4933a975c5a505474ccd6cb54e77 binary-source-provenance.json 21375103bda99d7465d8a29ee3929b2adb7e791abaa97322ab299f636eb9555f client-gemma3-p4096-r1-dense.txt 09f327b7653f31d7358e96004fdb801a15ff3ca3253194c1621e2f7f0d6dd1a4 client-gemma3-p4096-r1-paged-a.txt 602ef1cf6b0eab0d09fba8dec5bd40150ecf4e7475995501225710a077a844a5 client-gemma3-p4096-r1-paged-b.txt @@ -66,6 +66,7 @@ e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 complete 104f171b546eca3f1a8f8f2d0ab7fdba23a77d77c3cffae60ce9e36f910f2991 environment.txt 15a50c2718c3134750bf57e3195d294ea783cd7ae2f42940f6be1ad2dd63a607 gate.py ea84f12f4315e03456837f262234b06a1e9298b597378795e591623c20251912 hostgate.jsonl +037e0fb240ec19853a63eecd66a6d4d7045c510e4bd0f3e13ab558b48b13e31c release-build.log d7e0d1890ffc70338e714fd4f8ad07497ee7b8ce9edad0cf86012736b5db187a run.log 3b5816d50ae7cb4e5d5a5425cd31cadbcb05f33b57111c91f8e4edf35f994854 run.sh ec628e423b2742f5fb0e0f9c26e53413b23ce215796e20f3ae38e1631257df3e schedule.tsv diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/binary-source-provenance.json b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/binary-source-provenance.json index 5c3ace7c1..97c806bab 100644 --- a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/binary-source-provenance.json +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/binary-source-provenance.json @@ -8,7 +8,7 @@ "binary_sha256": "0cc49217021dab12b5add25dc2bb8de7e90c405c0b406607b244a8060d3ed800", "binary_size_bytes": 108106312, "binary_mtime_utc": "2026-10-11T03:25:49Z", - "release_build_log": "/tmp/mlxcel-auto-plan/2230-release-build.log", + "release_build_log": "docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/release-build.log", "release_build_log_sha256": "037e0fb240ec19853a63eecd66a6d4d7045c510e4bd0f3e13ab558b48b13e31c", "release_build_result": "Finished release profile in 19m 43s", "ancestor_command": "git merge-base --is-ancestor 2cb92404b153897440c4906a5548dcfb4889a9ad 35da2dbe32c731ff6645b2b32a2117af2965f332", @@ -21,6 +21,6 @@ "docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/test_protocol.py" ], "production_source_equivalent": true, - "derivation": "The preserved release-build session identifies 2cb92404 as the binary build source. That commit is an ancestor of the clean measurement checkout, and the complete intervening diff contains only the four listed benchmark-protocol files. No Rust, C++, model, or build-definition path changed between the binary source and measurement checkout.", - "limitation": "The release binary does not embed a source commit. The binary-source association is reconstructed after the run from the preserved build session, artifact hash and mtime, commit ancestry, and exact intervening path set; it is not a contemporaneous field in environment.txt. The original environment and measured scripts remain byte-for-byte unchanged." + "derivation": "The preserved release-build session context identifies 2cb92404 as the binary build source. That commit is an ancestor of the clean measurement checkout, and the complete intervening diff contains only the four listed benchmark-protocol files. No Rust, C++, model, or build-definition path changed between the binary source and measurement checkout.", + "limitation": "The release binary does not embed a source commit, and release-build.log proves the successful 19m43s build but does not print HEAD. The binary-source association is reconstructed after the run from the preserved session context, artifact hash and mtime, commit ancestry, and exact intervening path set; it is not a contemporaneous field in environment.txt. The original environment and measured scripts remain byte-for-byte unchanged." } diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/release-build.log b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/release-build.log new file mode 100644 index 000000000..2c4b96438 --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/release-build.log @@ -0,0 +1,128 @@ + Compiling ring v0.17.14 + Compiling rustls v0.23.45 + Compiling esaxx-rs v0.1.10 + Compiling onig_sys v69.9.3 + Compiling mlxcel-core v0.7.0 (/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/src/lib/mlxcel-core) + Compiling link-cplusplus v1.0.12 + Compiling onig v6.5.3 + Compiling rustls-webpki v0.103.14 + Compiling mime_guess v2.0.5 + Compiling tokenizers v0.22.2 + Compiling cxx v1.0.202 + Compiling tokio-rustls v0.26.5 + Compiling mlxcel-surgery v0.7.0 (/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/src/lib/mlxcel-surgery) + Compiling hyper-rustls v0.27.9 + Compiling ureq v3.4.0 + Compiling tokenizers v0.21.4 + Compiling reqwest v0.12.28 + Compiling rust-embed-impl v8.12.0 + Compiling sentencepiece-sys v0.13.2 + Compiling mlxcel v0.7.0 (/home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230) + Compiling blake3 v1.8.7 + Compiling toktrie_hf_tokenizers v1.8.0 + Compiling hf-hub v0.5.0 + Compiling rust-embed v8.12.0 + Compiling sentencepiece v0.13.2 +warning: mlxcel-core@0.7.0: In file included from cpp/mlx_cxx_bridge.cpp:7: +warning: mlxcel-core@0.7.0: /home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/target/release/build/mlxcel-core-4c8821fbbcc3d8f4/out/build/include/mlx/primitives.h: In member function 'std::pair mlx::core::AddMM::state() const': +warning: mlxcel-core@0.7.0: /home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/target/release/build/mlxcel-core-4c8821fbbcc3d8f4/out/build/include/mlx/primitives.h:204:41: note: parameter passing for argument of type 'std::pair' when C++17 is enabled changed to match C++14 in GCC 10.1 +warning: mlxcel-core@0.7.0: 204 | std::pair state() const { +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp: In function 'mlx_cxx::{anonymous}::get_compiled_q_path_proportional(int, int, int, float)::&)>': +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:3842:31: warning: '*(int*)((char*)&in_shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 3842 | int B = in_shape[0]; +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:3841:18: note: 'in_shape' declared here +warning: mlxcel-core@0.7.0: 3841 | auto in_shape = q_in.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:3842:31: warning: '*(int*)((char*)&in_shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 3842 | int B = in_shape[0]; +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:3841:18: note: 'in_shape' declared here +warning: mlxcel-core@0.7.0: 3841 | auto in_shape = q_in.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp: In lambda function: +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:2181:24: warning: '*(int*)((char*)&shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 2181 | int B = shape[0], H = shape[1], S = shape[2], D = shape[3]; +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:2180:14: note: 'shape' declared here +warning: mlxcel-core@0.7.0: 2180 | auto shape = x.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:2181:24: warning: '*(int*)((char*)&shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 2181 | int B = shape[0], H = shape[1], S = shape[2], D = shape[3]; +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_bridge.cpp:2180:14: note: 'shape' declared here +warning: mlxcel-core@0.7.0: 2180 | auto shape = x.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp: In lambda function: +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:3362:24: note: parameter passing for argument of type 'std::pair' when C++17 is enabled changed to match C++14 in GCC 10.1 +warning: mlxcel-core@0.7.0: 3362 | ref_rms); +warning: mlxcel-core@0.7.0: | ^ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp: At global scope: +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:3792:24: warning: 'mlx_cxx::{anonymous}::ADD3_LN_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 3792 | static const char* ADD3_LN_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:3027:24: warning: 'mlx_cxx::{anonymous}::MOE_FC1_RELU2_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 3027 | static const char* MOE_FC1_RELU2_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:2847:24: warning: 'mlx_cxx::{anonymous}::MOE_DOWN_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 2847 | static const char* MOE_DOWN_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:2708:24: warning: 'mlx_cxx::{anonymous}::MOE_GATEUP_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 2708 | static const char* MOE_GATEUP_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:2129:24: warning: 'mlx_cxx::{anonymous}::MAMBA1_SCAN_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 2129 | static const char* MAMBA1_SCAN_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:752:24: warning: 'mlx_cxx::{anonymous}::SSM_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 752 | static const char* SSM_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:452:24: warning: 'mlx_cxx::{anonymous}::XIELU_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 452 | static const char* XIELU_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_kernels.cpp:159:24: warning: 'mlx_cxx::{anonymous}::BITLINEAR_HIP_SOURCE' defined but not used [-Wunused-variable] +warning: mlxcel-core@0.7.0: 159 | static const char* BITLINEAR_HIP_SOURCE = R"( +warning: mlxcel-core@0.7.0: | ^~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp: In function 'void mlx_cxx::nemotron_decode_step(uint64_t, const MlxArray&, rust::cxxbridge1::Slice, rust::cxxbridge1::Slice, rust::cxxbridge1::Slice, rust::cxxbridge1::Slice, rust::cxxbridge1::Slice, std::unique_ptr&, rust::cxxbridge1::Slice >, rust::cxxbridge1::Slice >)': +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:156:16: warning: '*(int*)((char*)&id_shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 156 | h = reshape(h, {id_shape[0], id_shape[1], (int)h.shape().back()}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:155:10: note: 'id_shape' declared here +warning: mlxcel-core@0.7.0: 155 | auto id_shape = input_ids.inner.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:156:16: warning: '*(int*)((char*)&id_shape + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 156 | h = reshape(h, {id_shape[0], id_shape[1], (int)h.shape().back()}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:155:10: note: 'id_shape' declared here +warning: mlxcel-core@0.7.0: 155 | auto id_shape = input_ids.inner.shape(); +warning: mlxcel-core@0.7.0: | ^~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:244:28: warning: '*(int*)((char*)&ks + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 244 | k = reshape(k, {ks[0], ks[1], 1, ks[2], ks[3]}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:243:22: note: 'ks' declared here +warning: mlxcel-core@0.7.0: 243 | auto ks = k.shape(); +warning: mlxcel-core@0.7.0: | ^~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:244:28: warning: '*(int*)((char*)&ks + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 244 | k = reshape(k, {ks[0], ks[1], 1, ks[2], ks[3]}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:243:22: note: 'ks' declared here +warning: mlxcel-core@0.7.0: 243 | auto ks = k.shape(); +warning: mlxcel-core@0.7.0: | ^~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:248:28: warning: '*(int*)((char*)&vs + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 248 | v = reshape(v, {vs[0], vs[1], 1, vs[2], vs[3]}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:247:22: note: 'vs' declared here +warning: mlxcel-core@0.7.0: 247 | auto vs = v.shape(); +warning: mlxcel-core@0.7.0: | ^~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:248:28: warning: '*(int*)((char*)&vs + offsetof(mlx::core::SmallVector >,mlx::core::SmallVector >::inline_storage_[0]))' may be used uninitialized [-Wmaybe-uninitialized] +warning: mlxcel-core@0.7.0: 248 | v = reshape(v, {vs[0], vs[1], 1, vs[2], vs[3]}); +warning: mlxcel-core@0.7.0: | ~~~~~~~^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +warning: mlxcel-core@0.7.0: cpp/mlx_cxx_nemotron.cpp:247:22: note: 'vs' declared here +warning: mlxcel-core@0.7.0: 247 | auto vs = v.shape(); +warning: mlxcel-core@0.7.0: | ^~ +warning: mlxcel-core@0.7.0: In file included from ../mlx-cpp/turbo/kv_inplace_write.cpp:23: +warning: mlxcel-core@0.7.0: /home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/target/release/build/mlxcel-core-4c8821fbbcc3d8f4/out/build/include/mlx/primitives.h: In member function 'std::pair mlx::core::AddMM::state() const': +warning: mlxcel-core@0.7.0: /home/inureyes/Development/backend.ai/wt-auto-20261011T0224-issue-2230/target/release/build/mlxcel-core-4c8821fbbcc3d8f4/out/build/include/mlx/primitives.h:204:41: note: parameter passing for argument of type 'std::pair' when C++17 is enabled changed to match C++14 in GCC 10.1 +warning: mlxcel-core@0.7.0: 204 | std::pair state() const { +warning: mlxcel-core@0.7.0: | ^ + Finished `release` profile [optimized] target(s) in 19m 43s diff --git a/docs/benchmark_results/model-owned-storage-gb10-2026-10-11.md b/docs/benchmark_results/model-owned-storage-gb10-2026-10-11.md index 730f6d1ef..ce8c3fbb6 100644 --- a/docs/benchmark_results/model-owned-storage-gb10-2026-10-11.md +++ b/docs/benchmark_results/model-owned-storage-gb10-2026-10-11.md @@ -18,7 +18,7 @@ This decision concerns storage allocation and the per-step model-owned KV mirror The committed [raw evidence](data/model-owned-storage-gb10-2026-10-11/README.md) includes the exact contemporaneous environment, measured script hashes, schedule, host-gate snapshots, server and client output, analyzer output, completion marker, checksums, and a separate derived binary-source proof. -`environment.txt` uses `commit` for the clean checkout that ran the sweep. The release binary does not embed a source revision, so its association with `2cb92404` was reconstructed after the run from the preserved release-build session, the artifact hash and mtime, and Git history. Commit `2cb92404` is an ancestor of `35da2dbe`, and the complete intervening diff contains only this run's `analyze.py`, `gate.py`, `run.sh`, and `test_protocol.py`. No Rust, C++, model, or build-definition path changed. The original environment and measured script bytes and hashes remain unchanged. [The companion provenance record](data/model-owned-storage-gb10-2026-10-11/binary-source-provenance.json) records the commands, exact paths, release-log hash, and limitation. +`environment.txt` uses `commit` for the clean checkout that ran the sweep. The release binary does not embed a source revision, so its association with `2cb92404` was reconstructed after the run from the preserved release-build session context, the artifact hash and mtime, and Git history. The committed release log proves the successful 19m43s build but does not print HEAD. Commit `2cb92404` is an ancestor of `35da2dbe`, and the complete intervening diff contains only this run's `analyze.py`, `gate.py`, `run.sh`, and `test_protocol.py`. No Rust, C++, model, or build-definition path changed. The original environment and measured script bytes and hashes remain unchanged. [The companion provenance record](data/model-owned-storage-gb10-2026-10-11/binary-source-provenance.json) records the commands, exact paths, release-log hash, and limitation. ## Method From 9b53622274f5880c69173496c8bd4037b8179f17 Mon Sep 17 00:00:00 2001 From: Jeongkyu Shin Date: Sun, 11 Oct 2026 16:24:20 +0900 Subject: [PATCH 6/7] chore: scope raw evidence whitespace checks Preserve the measured client, SSM stage, and shell-command records byte-for-byte while disabling only their known blank-EOF or trailing-space checks. Refresh both evidence manifests to cover the scoped attributes. Validation: both manifests verify and the branch diff passes Git's whitespace check. Refs #2230 --- .../data/model-owned-storage-gb10-2026-10-11/.gitattributes | 1 + .../data/model-owned-storage-gb10-2026-10-11/SHA256SUMS | 1 + .../data/ssm-bf16-parity-gb10-2026-10-11/.gitattributes | 2 ++ .../data/ssm-bf16-parity-gb10-2026-10-11/SHA256SUMS | 1 + 4 files changed, 5 insertions(+) create mode 100644 docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/.gitattributes create mode 100644 docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/.gitattributes diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/.gitattributes b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/.gitattributes new file mode 100644 index 000000000..199e94c3e --- /dev/null +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/.gitattributes @@ -0,0 +1 @@ +client-*.txt whitespace=-blank-at-eof diff --git a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/SHA256SUMS b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/SHA256SUMS index a15edd26e..c1712d390 100644 --- a/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/SHA256SUMS +++ b/docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/SHA256SUMS @@ -1,3 +1,4 @@ +0e3cded960823ef682eb6bbf1ea3673811690c5b5db6e1147a0d162a74f5278a .gitattributes a7375a6a8077e73cc4afca3aa8125072eeffe4125821410f7e83ae4ffccb18cb README.md 6e27465a51285608dbbd38798864556ee8b8cca4bf2c422fb965e45dfd60bce7 analysis.txt f17233d3e5b5339424c050404ea81a171ad43b0abd74ebb4d4ddc681f894f3f8 analyze.py diff --git a/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/.gitattributes b/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/.gitattributes new file mode 100644 index 000000000..1f90ce1d3 --- /dev/null +++ b/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/.gitattributes @@ -0,0 +1,2 @@ +stage-deviations.log whitespace=-blank-at-eof +granite-parity/provenance.txt whitespace=-trailing-space diff --git a/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/SHA256SUMS b/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/SHA256SUMS index 9d7fd105c..006da0eb5 100644 --- a/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/SHA256SUMS +++ b/docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/SHA256SUMS @@ -1,3 +1,4 @@ +5558373ffa3d9df14357938fd908b2c8888cb6a98b6c6d48b5e935c416e2acd7 .gitattributes 14b73bcdf4de9f1361bb44249e184d1a032216908052a28a92bcdaa418a0f4ae README.md aabe38700bc3adfd7bb832d211b0eac5bf7342f4e476490ab6d551cc697c4b76 granite-parity/comparison.json 726bf4f6ad48e396b97c8e109f34f84bf103c44aa8a15ce7d790e22e75eefd2a granite-parity/graph.stderr From bfdc46f8dbfd3805a1eebb90c038a5974792a90a Mon Sep 17 00:00:00 2001 From: Jeongkyu Shin Date: Sun, 11 Oct 2026 16:28:28 +0900 Subject: [PATCH 7/7] docs: add PR 2270 technical reports Record the compatibility removal, paired/null storage decision, SSM precision diagnosis, raw provenance boundary, and final CUDA/Granite validation in English and Korean. Refs #2230, #2270 --- ...post-epic-paged-ssm-cleanup-20261011.en.md | 130 ++++++++++++++++++ ...post-epic-paged-ssm-cleanup-20261011.ko.md | 130 ++++++++++++++++++ 2 files changed, 260 insertions(+) create mode 100644 TECHNICAL_REPORTS/2270-post-epic-paged-ssm-cleanup-20261011.en.md create mode 100644 TECHNICAL_REPORTS/2270-post-epic-paged-ssm-cleanup-20261011.ko.md diff --git a/TECHNICAL_REPORTS/2270-post-epic-paged-ssm-cleanup-20261011.en.md b/TECHNICAL_REPORTS/2270-post-epic-paged-ssm-cleanup-20261011.en.md new file mode 100644 index 000000000..2113c3a8a --- /dev/null +++ b/TECHNICAL_REPORTS/2270-post-epic-paged-ssm-cleanup-20261011.en.md @@ -0,0 +1,130 @@ +# Technical Report: PR #2270 - Post-epic paged and SSM cleanup + +**Date**: 2026-10-11 +**Status**: Completed and verified on CUDA (GB10) +**Languages**: C++, Rust, Python, Bash, Markdown +**Risk Level**: Medium + +## Executive summary + +PR #2270 completes the three post-epic tasks in issue #2230. It removes the unused dense-pointer and rotating paged compatibility surface, replaces its role in pooled decode tests with an independent float32 attention oracle, resolves the Gemma 3 and Llama 4 model-owned storage flags through a complete paired/null serving sweep, and fixes CUDA bf16 SSM parity at the first numerically divergent stage. + +The SSM defect was caused by the pinned MLX type-promotion table selecting bf16 for `float32 + bfloat16`. `compute_dt_compiled` cast `dt` to float32 but left `dt_bias` in bf16, so the compiled softplus result was rounded before any backend kernel received it. Casting both operands to float32 restores the documented recurrence for Metal, CUDA, and HIP without changing a tolerance. A deterministic Granite checkpoint comparison produced the same 64 generated token IDs through the fused CUDA kernel and graph paths. + +The storage sweep ran 60 fresh-server arms and 180 successful client cells. None of the six prompt/concurrency cells for either family cleared the repeated-paged null spread in either direction, so both `supports_paged_decode_backend()` flags remain `true`. + +## 1. Problem statement + +### 1.1 Obsolete compatibility surface + +After #2210 and #2217, production models no longer called the dense-pointer or rotating paged decode compatibility routes. Their C++ entry points, CXX bridge declarations, Rust wrappers and validators, cache metadata, and direct FFI tests remained only to support each other. Four active pooled-kernel tests still used the old dense fallback as their oracle, preventing removal until those tests had an independent reference. + +### 1.2 Unresolved model-owned storage default + +Gemma 3 and Llama 4 own their KV rows and compute attention from dense storage, while their advertised paged support still causes a multi-slot server to allocate paged storage and mirror each decode step into it. Earlier Gemma 3 numbers overlapped their run-to-run spread, and Llama 4 had no measurement. The flags therefore required a per-family decision based on paired dense-versus-paged deltas and a repeated-paged null arm. + +### 1.3 CUDA bf16 SSM parity failure + +The Granite bf16 SSM cases failed against the float32 graph reference with normalized state RMS around `1.8e-3` to `2.5e-3`, while f32 cases and the Nemotron-H bf16-activation/f32-`A_log` case passed. The same failure existed before the epic. The acceptance criteria required measuring `dt'`, decay, and state before changing production code and prohibited widening the existing tolerance without a derivation. + +## 2. Implementation + +### 2.1 Compatibility removal and independent oracle + +The PR deletes the dense and rotating compatibility C++ implementations and declarations, their bridge entries, Rust fallback wrappers and validators, `PagedDecodeMetadata`, `RotatingPagedDecodeMetadata`, and tests that existed only for those APIs. The pooled empty-batch case remains under its current pooled behavior. + +The four pooled decode tests now use a test-only `dense_reference_decode_attention` helper. It converts query, key, and value tensors to float32, repeats KV heads for grouped-query attention, computes `softmax(q k^T * scale) v`, and concatenates the reference rows. Each test preserves its original absolute RMS `<5e-3` assertion and adds normalized RMS `<5e-3`. A temporary `1.01` multiplication of the pooled output made all four tests fail near normalized RMS `0.010`; the mutation was removed before commit. + +### 2.2 Model-owned storage measurement + +The benchmark starts a fresh `mlxcel serve --parallel 8` process for every arm. It covers two models, 512- and 4096-token prompts, five rounds, and `paged-a`, `dense`, and `paged-b` arms in rotating order. Each arm measures concurrency 1, 4, and 8 for 128 generated tokens. The outer command holds `gpu-lock`; an inner fail-closed gate requires a sustained quiet host, a free port before and after the wait, no CI/build/foreign-model activity, sufficient memory, and clean NVIDIA driver counters. + +The analyzer refuses to calculate a decision without the completion marker, clean checkout and artifact provenance, matching configured and built MLX commits, exact measured script hashes, the ordered 60-arm schedule, 61 accepted quiet-gate records, 60 nonempty backend-marked server logs, and exactly 180 successful client keys. `paged-a` is the paired baseline, while `paged-b / paged-a - 1` defines the null spread. Dense wins a cell only when its median paired delta exceeds `max(0, null_high)`; paged wins only below `min(0, null_low)`. + +### 2.3 Shared SSM precision correction + +The first large divergence appeared in compiled `dt'`, whose output dtype was bf16. The pinned MLX promotion table explained why: the graph added float32 `dt` to bf16 `dt_bias`, and the result followed the table's bf16 entry. The PR explicitly casts `dt_bias` to float32 beside `dt` before the compiled add, softplus, and clip. + +`compute_dt_compiled` runs before backend dispatch. The same correction therefore supplies float32 `dt'` semantics to Metal, CUDA, and HIP without duplicating backend-specific logic. The CUDA recurrence kernel, float32 reference, and tolerances remain unchanged. + +## 3. Measured results + +### 3.1 SSM intermediate deviations + +For Granite seed 2069, normalized RMS/max deviations against the stable float32 graph were: + +| Stage | Before | After | +|---|---:|---:| +| Compiled `dt'` | `1.957140831e-3 / 5.875451502e-3` | `2.890811210e-8 / 2.002812756e-7` | +| Compiled decay | `3.008783834e-3 / 1.075881018e-2` | `6.809246822e-10 / 4.717584582e-9` | +| Kernel state | `2.532402801e-3 / 4.127457397e-2` | `2.838613609e-8 / 1.701009653e-6` | + +The correction reduces the downstream deviation by roughly five orders of magnitude and brings both formerly failing CUDA tests inside their existing budgets. + +### 3.2 Storage sweep + +Rates below are aggregate generated tokens per second. The paired column shows the five-round median and full paired range; the null column is the repeated-paged range. + +| Model | Prompt | C | paged-a | dense | paged-b | Dense paired delta | Null spread | Decision | +|---|---:|---:|---:|---:|---:|---:|---:|---| +| Gemma 3 | 512 | 1 | 154.4 | 154.0 | 154.0 | +0.52% (-2.76..+3.63) | -2.37..+3.84% | unresolved | +| Gemma 3 | 512 | 4 | 361.0 | 363.6 | 360.6 | +1.29% (-2.11..+2.49) | -1.35..+3.52% | unresolved | +| Gemma 3 | 512 | 8 | 194.9 | 197.2 | 194.4 | +1.16% (+0.25..+6.37) | -0.88..+3.80% | unresolved | +| Gemma 3 | 4096 | 1 | 114.6 | 114.4 | 113.0 | -0.18% (-1.82..+0.26) | -2.33..+1.31% | unresolved | +| Gemma 3 | 4096 | 4 | 219.4 | 216.6 | 218.6 | -0.60% (-2.74..+0.54) | -3.45..+0.50% | unresolved | +| Gemma 3 | 4096 | 8 | 184.4 | 185.8 | 184.0 | +0.49% (-0.53..+1.98) | -3.13..+1.91% | unresolved | +| Llama 4 | 512 | 1 | 11.7 | 11.8 | 11.8 | +0.85% (+0.00..+83.08) | -0.85..+81.54% | unresolved | +| Llama 4 | 512 | 4 | 31.0 | 30.8 | 30.7 | +0.65% (-3.53..+2.61) | -0.97..+3.53% | unresolved | +| Llama 4 | 512 | 8 | 17.8 | 17.9 | 17.7 | +0.00% (-1.69..+13.29) | -1.13..+10.76% | unresolved | +| Llama 4 | 4096 | 1 | 8.0 | 8.0 | 8.0 | +1.25% (+0.00..+1.27) | +0.00..+2.53% | unresolved | +| Llama 4 | 4096 | 4 | 22.9 | 23.1 | 23.0 | +0.44% (-0.43..+2.25) | -2.99..+4.05% | unresolved | +| Llama 4 | 4096 | 8 | 14.7 | 14.7 | 14.5 | +0.68% (-0.68..+3.55) | -2.03..+2.84% | unresolved | + +No cell satisfies either clear-win rule. `auto` on a multi-slot server and an explicit `paged` request continue to select paged storage for both model families, including pool allocation and the per-step mirror. Their attention path continues to use dense rows. + +### 3.3 Observed first-arm anomaly + +The first Llama 4 512-token round began with `paged-a`. At concurrency 1, that arm recorded 6.5 aggregate tok/s and 12690.4 ms mean TTFT. The adjacent dense and repeated-paged arms recorded 11.9 and 11.8 aggregate tok/s with 4072.6 and 4128.2 ms TTFT. Rounds 2 through 5 were approximately 11.7 to 11.9 aggregate tok/s across all arms. + +The first round therefore contributes a +83.08% dense paired delta and +81.54% null delta. Every record remains in the result. The host gate accepted the arm, but the evidence does not establish its cause; a cold or warm-up effect is only a hypothesis. The run was not filtered or repeated after inspection. Under the registered rule, the paired median is +0.85% and does not clear the +81.54% null upper bound. + +## 4. Verification + +| Area | Result | +|---|---| +| Removed compatibility expression across `src/` | No matches | +| Four focused pooled CUDA tests | Passed | +| Temporary pooled-output `1.01` mutation | All four tests failed; mutation removed | +| Focused CUDA SSM parity | Five passed, including both former failures | +| Granite real checkpoint | Fused kernel and graph produced identical 64/64 greedy token IDs | +| Granite token sequence SHA-256 | `9df6f4fc2abbfcb121fa5dca3ad1e140e8db5b67d28118a905b93022f8905829` | +| CUDA release build | Passed in 19m43s | +| Full `make verify-test-cuda` under GPU lock | Exit 0, zero failures | +| Workspace/all-target CUDA clippy and formatting | Passed | +| Dtype-key, port-dispatch, and llama-compat contracts | Passed | +| Storage analyzer | Validated 60 arms and 180 successful cells | +| Benchmark protocol fixtures | 9 passed with warnings treated as errors | +| Raw storage and SSM manifests | Passed | +| Independent implementation, security, and evidence reviews | No remaining findings | + +The Granite comparison used prompt `Explain the Mamba state update.`, 64 greedy tokens, temperature 0, and deterministic SDPA. The graph arm set `MLXCEL_SSM_CUDA_KERNEL=0`. + +## 5. Provenance and limitations + +The storage run's contemporaneous `environment.txt` records clean benchmark checkout `35da2dbe`. The measured release binary was built from production commit `2cb92404`; the binary does not embed a commit, so this association is reconstructed from the preserved build-session context, artifact hash and mtime, ancestry, and the exact intervening path set. `2cb92404` is an ancestor of `35da2dbe`, and all intervening changes are confined to the four benchmark protocol files. No Rust, C++, model, or build-definition source differs. The original environment and measured script bytes remain unchanged, while a separate derived provenance record documents the proof and its limitation. + +The committed release log proves that the release build completed in 19m43s but does not print HEAD. The raw package retains all client/server output, the 61 accepted host-gate records, ordered schedule, analyzer result, completion marker, script hashes, MLX pin, derived provenance, SSM stage output, Granite comparison, and checksums. + +## 6. Compatibility, security, and performance + +The removed APIs had no production callers. The SSM fix changes numeric preprocessing only where a bf16 bias previously caused unintended rounding and adds no per-token host synchronization or new dispatch. No new runtime dependency or public option is introduced. + +Both model-owned flags remain enabled, so serving behavior is stable. The benchmark provides no clear throughput winner within its observed repeatability. Independent security review found no production-code issue. Benchmark review initially found fail-open host/provenance handling; the final gate and analyzer reject nonquiet, incomplete, reordered, dirty, or backend-substituted evidence before calculation. + +## 7. Related records + +- Issue #2230: post-epic cleanup requirements +- PR #2270: implementation and evidence +- `docs/benchmark_results/model-owned-storage-gb10-2026-10-11.md`: dated storage decision +- `docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/`: complete storage protocol and raw output +- `docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/`: original stage deviations and Granite parity artifacts diff --git a/TECHNICAL_REPORTS/2270-post-epic-paged-ssm-cleanup-20261011.ko.md b/TECHNICAL_REPORTS/2270-post-epic-paged-ssm-cleanup-20261011.ko.md new file mode 100644 index 000000000..3f4b283c1 --- /dev/null +++ b/TECHNICAL_REPORTS/2270-post-epic-paged-ssm-cleanup-20261011.ko.md @@ -0,0 +1,130 @@ +# 기술 보고서: PR #2270 - 에픽 이후 paged 및 SSM 정리 + +**날짜**: 2026-10-11 +**상태**: CUDA(GB10)에서 구현 및 검증 완료 +**언어**: C++, Rust, Python, Bash, Markdown +**위험 수준**: 중간 + +## 요약 + +PR #2270은 이슈 #2230의 에픽 이후 작업 세 가지를 완료합니다. 사용하지 않는 dense-pointer 및 rotating paged 호환 표면을 제거하고, pooled decode 테스트에서 그 역할을 독립적인 float32 attention oracle로 교체했습니다. 또한 완전한 paired/null serving sweep으로 Gemma 3와 Llama 4의 model-owned storage 플래그를 결정하고, 수치가 처음 어긋나는 단계에서 CUDA bf16 SSM parity를 수정했습니다. + +SSM 결함의 원인은 고정된 MLX type-promotion 표가 `float32 + bfloat16` 결과를 bf16으로 선택하는 데 있었습니다. `compute_dt_compiled`는 `dt`를 float32로 변환했지만 `dt_bias`는 bf16으로 남겨 두었으므로, 어떤 backend kernel이 값을 받기 전에 compiled softplus 결과가 반올림되었습니다. 두 피연산자를 모두 float32로 변환하여 허용 오차를 바꾸지 않고 Metal, CUDA, HIP에서 문서화된 recurrence를 복원했습니다. 실제 Granite checkpoint의 deterministic 비교에서는 fused CUDA kernel과 graph path가 생성한 64개 token ID가 모두 같았습니다. + +Storage sweep은 fresh server 60개 arm과 성공한 client cell 180개를 실행했습니다. 두 family 모두 여섯 prompt/concurrency cell 중 어느 것도 repeated-paged null spread를 어느 방향으로도 벗어나지 못했으므로 두 `supports_paged_decode_backend()` 플래그는 계속 `true`입니다. + +## 1. 문제 + +### 1.1 오래된 호환 표면 + +#2210과 #2217 이후 production model은 dense-pointer 또는 rotating paged decode 호환 경로를 더 이상 호출하지 않았습니다. C++ entry point, CXX bridge declaration, Rust wrapper와 validator, cache metadata, 직접 FFI test는 서로를 유지하기 위해서만 남아 있었습니다. 네 개의 활성 pooled-kernel test가 기존 dense fallback을 oracle로 사용하고 있었으므로, 독립 reference를 먼저 마련해야 이 코드를 제거할 수 있었습니다. + +### 1.2 미결정 model-owned storage 기본값 + +Gemma 3와 Llama 4는 자체 KV row를 소유하고 dense storage에서 attention을 계산합니다. 그러나 paged 지원을 알리는 플래그 때문에 multi-slot server는 paged storage를 할당하고 각 decode step을 여기에 mirror합니다. 기존 Gemma 3 수치는 실행 간 분산과 겹쳤고 Llama 4 측정은 없었습니다. 따라서 family별 플래그는 paired dense-versus-paged delta와 repeated-paged null arm으로 결정해야 했습니다. + +### 1.3 CUDA bf16 SSM parity 실패 + +Granite bf16 SSM case는 float32 graph reference 대비 normalized state RMS 약 `1.8e-3`에서 `2.5e-3`으로 실패했지만, f32 case와 bf16 activation/f32 `A_log`를 쓰는 Nemotron-H case는 통과했습니다. 같은 실패가 에픽 이전에도 존재했습니다. Acceptance criteria는 production code를 바꾸기 전에 `dt'`, decay, state를 측정하고 근거 없이 tolerance를 넓히지 않도록 요구했습니다. + +## 2. 구현 + +### 2.1 호환 경로 제거와 독립 oracle + +PR은 dense 및 rotating 호환 C++ 구현과 declaration, bridge entry, Rust fallback wrapper와 validator, `PagedDecodeMetadata`, `RotatingPagedDecodeMetadata`, 그리고 해당 API만 검증하던 test를 삭제합니다. Pooled empty-batch case는 현재 pooled 동작을 대상으로 유지합니다. + +네 pooled decode test는 이제 test 전용 `dense_reference_decode_attention` helper를 사용합니다. 이 helper는 query, key, value를 float32로 변환하고 grouped-query attention을 위해 KV head를 반복한 뒤 `softmax(q k^T * scale) v`를 계산하고 reference row를 연결합니다. 각 test는 원래의 absolute RMS `<5e-3` assertion을 유지하고 normalized RMS `<5e-3`를 추가합니다. Pooled output에 임시로 `1.01`을 곱하면 네 test가 모두 normalized RMS 약 `0.010`으로 실패했으며, mutation은 commit 전에 제거했습니다. + +### 2.2 Model-owned storage 측정 + +Benchmark는 arm마다 fresh `mlxcel serve --parallel 8` process를 시작합니다. 두 모델, 512 및 4096 token prompt, 다섯 round, 회전 순서의 `paged-a`, `dense`, `paged-b` arm을 측정합니다. 각 arm은 128개 token 생성에 대해 concurrency 1, 4, 8을 실행합니다. 바깥 command는 `gpu-lock`을 유지하고, 안쪽 fail-closed gate는 지속적인 quiet-host 구간, wait 전후의 빈 port, CI/build/foreign-model activity 없음, 충분한 memory, 깨끗한 NVIDIA driver counter를 요구합니다. + +Analyzer는 completion marker, clean checkout 및 artifact provenance, 일치하는 configured/built MLX commit, 측정 당시 script hash, 순서가 정확한 60-arm schedule, 승인된 quiet-gate record 61개, backend marker가 있는 nonempty server log 60개, 정확히 180개의 성공한 client key가 없으면 decision을 계산하지 않습니다. `paged-a`가 paired baseline이고 `paged-b / paged-a - 1`이 null spread입니다. Dense는 median paired delta가 `max(0, null_high)`보다 클 때만 cell에서 이기고, paged는 `min(0, null_low)`보다 작을 때만 이깁니다. + +### 2.3 공유 SSM 정밀도 수정 + +첫 번째 큰 편차는 compiled `dt'`에서 나타났으며 output dtype은 bf16이었습니다. 고정된 MLX promotion 표가 원인을 설명했습니다. Graph는 float32 `dt`와 bf16 `dt_bias`를 더했고, 결과가 표의 bf16 항목을 따랐습니다. PR은 compiled add, softplus, clip 전에 `dt_bias`를 `dt`와 함께 명시적으로 float32로 변환합니다. + +`compute_dt_compiled`는 backend dispatch 전에 실행됩니다. 따라서 backend별 code를 복제하지 않고 Metal, CUDA, HIP 모두에 동일한 float32 `dt'` semantics를 제공합니다. CUDA recurrence kernel, float32 reference, tolerance는 변경하지 않았습니다. + +## 3. 측정 결과 + +### 3.1 SSM 중간 단계 편차 + +Granite seed 2069에서 안정적인 float32 graph 대비 normalized RMS/max 편차는 다음과 같습니다. + +| 단계 | 수정 전 | 수정 후 | +|---|---:|---:| +| Compiled `dt'` | `1.957140831e-3 / 5.875451502e-3` | `2.890811210e-8 / 2.002812756e-7` | +| Compiled decay | `3.008783834e-3 / 1.075881018e-2` | `6.809246822e-10 / 4.717584582e-9` | +| Kernel state | `2.532402801e-3 / 4.127457397e-2` | `2.838613609e-8 / 1.701009653e-6` | + +수정 후 downstream 편차가 약 다섯 자릿수 줄었고, 이전에 실패하던 두 CUDA test 모두 기존 budget 안으로 들어왔습니다. + +### 3.2 Storage sweep + +아래 속도는 aggregate generated token/s입니다. Paired 열은 다섯 round의 median과 전체 paired range이고 null 열은 repeated-paged range입니다. + +| 모델 | Prompt | C | paged-a | dense | paged-b | Dense paired delta | Null spread | 결정 | +|---|---:|---:|---:|---:|---:|---:|---:|---| +| Gemma 3 | 512 | 1 | 154.4 | 154.0 | 154.0 | +0.52% (-2.76..+3.63) | -2.37..+3.84% | unresolved | +| Gemma 3 | 512 | 4 | 361.0 | 363.6 | 360.6 | +1.29% (-2.11..+2.49) | -1.35..+3.52% | unresolved | +| Gemma 3 | 512 | 8 | 194.9 | 197.2 | 194.4 | +1.16% (+0.25..+6.37) | -0.88..+3.80% | unresolved | +| Gemma 3 | 4096 | 1 | 114.6 | 114.4 | 113.0 | -0.18% (-1.82..+0.26) | -2.33..+1.31% | unresolved | +| Gemma 3 | 4096 | 4 | 219.4 | 216.6 | 218.6 | -0.60% (-2.74..+0.54) | -3.45..+0.50% | unresolved | +| Gemma 3 | 4096 | 8 | 184.4 | 185.8 | 184.0 | +0.49% (-0.53..+1.98) | -3.13..+1.91% | unresolved | +| Llama 4 | 512 | 1 | 11.7 | 11.8 | 11.8 | +0.85% (+0.00..+83.08) | -0.85..+81.54% | unresolved | +| Llama 4 | 512 | 4 | 31.0 | 30.8 | 30.7 | +0.65% (-3.53..+2.61) | -0.97..+3.53% | unresolved | +| Llama 4 | 512 | 8 | 17.8 | 17.9 | 17.7 | +0.00% (-1.69..+13.29) | -1.13..+10.76% | unresolved | +| Llama 4 | 4096 | 1 | 8.0 | 8.0 | 8.0 | +1.25% (+0.00..+1.27) | +0.00..+2.53% | unresolved | +| Llama 4 | 4096 | 4 | 22.9 | 23.1 | 23.0 | +0.44% (-0.43..+2.25) | -2.99..+4.05% | unresolved | +| Llama 4 | 4096 | 8 | 14.7 | 14.7 | 14.5 | +0.68% (-0.68..+3.55) | -2.03..+2.84% | unresolved | + +어느 cell도 clear-win 조건을 충족하지 않습니다. Multi-slot server의 `auto`와 명시적인 `paged` request는 두 model family 모두에서 paged storage를 계속 선택하며, 여기에는 pool allocation과 per-step mirror가 포함됩니다. Attention path는 계속 dense row를 사용합니다. + +### 3.3 관찰된 첫 arm 이상값 + +Llama 4의 첫 512-token round는 `paged-a`로 시작했습니다. Concurrency 1에서 이 arm은 aggregate 6.5 tok/s와 mean TTFT 12690.4 ms를 기록했습니다. 이어진 dense와 repeated-paged arm은 aggregate 11.9 및 11.8 tok/s, TTFT 4072.6 및 4128.2 ms였습니다. Round 2부터 5까지는 모든 arm에서 약 11.7에서 11.9 aggregate tok/s였습니다. + +따라서 첫 round는 +83.08% dense paired delta와 +81.54% null delta를 만듭니다. 모든 record를 결과에 그대로 유지했습니다. Host gate는 arm을 승인했지만 evidence는 원인을 입증하지 않습니다. Cold 또는 warm-up 효과는 하나의 가설일 뿐입니다. 결과를 본 뒤 run을 필터링하거나 다시 실행하지 않았습니다. 사전 정의한 rule에서 paired median +0.85%는 null upper bound +81.54%를 넘지 못하므로 이 cell은 unresolved입니다. + +## 4. 검증 + +| 영역 | 결과 | +|---|---| +| `src/`의 제거 대상 호환 표현식 | 일치 항목 없음 | +| 네 focused pooled CUDA test | Passed | +| 임시 pooled-output `1.01` mutation | 네 test 모두 실패, mutation 제거 | +| Focused CUDA SSM parity | 이전 두 실패를 포함해 5 passed | +| 실제 Granite checkpoint | Fused kernel과 graph가 greedy token ID 64/64개 동일하게 생성 | +| Granite token sequence SHA-256 | `9df6f4fc2abbfcb121fa5dca3ad1e140e8db5b67d28118a905b93022f8905829` | +| CUDA release build | 19분 43초에 성공 | +| GPU lock 아래 전체 `make verify-test-cuda` | Exit 0, 실패 0 | +| Workspace/all-target CUDA clippy와 formatting | Passed | +| Dtype-key, port-dispatch, llama-compat contract | Passed | +| Storage analyzer | 60 arm과 성공한 180 cell 검증 | +| Benchmark protocol fixture | Warning을 error로 처리하고 9 passed | +| Raw storage 및 SSM manifest | Passed | +| 독립 구현, 보안, evidence review | 남은 finding 없음 | + +Granite 비교는 prompt `Explain the Mamba state update.`, greedy token 64개, temperature 0, deterministic SDPA를 사용했습니다. Graph arm은 `MLXCEL_SSM_CUDA_KERNEL=0`을 설정했습니다. + +## 5. Provenance와 한계 + +Storage run 당시의 `environment.txt`는 clean benchmark checkout `35da2dbe`를 기록합니다. 측정한 release binary는 production commit `2cb92404`에서 build했습니다. Binary에는 commit이 내장되어 있지 않으므로, 이 연결은 보존된 build-session context, artifact hash와 mtime, ancestry, 정확한 intervening path set으로 사후 재구성했습니다. `2cb92404`는 `35da2dbe`의 ancestor이고, 그 사이 변경은 네 benchmark protocol file로 제한됩니다. Rust, C++, model, build-definition source에는 차이가 없습니다. 원래 environment와 측정 당시 script byte는 그대로 유지하고, 별도의 derived provenance record가 증명과 한계를 기록합니다. + +Committed release log는 release build가 19분 43초에 완료되었음을 증명하지만 HEAD는 출력하지 않습니다. Raw package는 모든 client/server output, 승인된 host-gate record 61개, ordered schedule, analyzer result, completion marker, script hash, MLX pin, derived provenance, SSM stage output, Granite comparison, checksum을 보존합니다. + +## 6. 호환성, 보안, 성능 + +제거한 API에는 production caller가 없었습니다. SSM 수정은 bf16 bias가 의도하지 않은 반올림을 만들던 numeric preprocessing만 바꾸며 token당 host synchronization이나 새 dispatch를 추가하지 않습니다. 새로운 runtime dependency나 public option도 없습니다. + +두 model-owned flag가 계속 enabled이므로 serving 동작은 유지됩니다. Benchmark는 관찰된 반복성 범위 안에서 명확한 throughput 승자를 보여 주지 않습니다. 독립 security review는 production-code 문제를 찾지 못했습니다. Benchmark review는 처음에 fail-open host/provenance 처리를 발견했으며, 최종 gate와 analyzer는 계산 전에 nonquiet, incomplete, reordered, dirty, backend-substituted evidence를 거부합니다. + +## 7. 관련 기록 + +- 이슈 #2230: 에픽 이후 정리 요구 사항 +- PR #2270: 구현과 evidence +- `docs/benchmark_results/model-owned-storage-gb10-2026-10-11.md`: 날짜가 기록된 storage 결정 +- `docs/benchmark_results/data/model-owned-storage-gb10-2026-10-11/`: 완전한 storage protocol 및 raw output +- `docs/benchmark_results/data/ssm-bf16-parity-gb10-2026-10-11/`: 원래 stage 편차 및 Granite parity artifact