feat: i18n CN/EN, ops kernel labels, UI polish, compare improvements

CONTRIBUTING.md

@@ -14,7 +14,7 @@ in the leaderboard and submitting your results.
 # 1. Fork the repo on GitHub, then clone your fork
 git clone https://github.com/<you>/AccelMark.git
 cd AccelMark
-pip install -e .
+pip install "jsonschema>=4.0" "numpy>=1.24" "pyyaml>=6.0"   # framework dependencies (Python >=3.10 required)
 pip install -r runners/nvidia_vllm_47f5d58e/requirements.txt
 
 # 2. Set your name (one-time setup)

README.md

@@ -51,7 +51,7 @@
 # 1. Clone and install
 git clone https://github.com/FreedomIntelligence/AccelMark.git
 cd AccelMark
-pip install -e .                                              # installs framework dependencies (Python >=3.10 required)
+pip install "jsonschema>=4.0" "numpy>=1.24" "pyyaml>=6.0"       # framework dependencies (Python >=3.10 required)
 pip install -r runners/nvidia_vllm_47f5d58e/requirements.txt # installs runner dependencies
 
 # 2. One-time setup
@@ -118,6 +118,7 @@ Reference runners live under `runners/` (see each folder’s `meta.json`). The t
 | NVIDIA GPU | `nvidia_vllm_47f5d58e` | vLLM | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
 | NVIDIA V100 (SM70) | `nvidia_onecat_vllm_12a253c2` | 1Cat-vLLM | ⋯ | ⋯ | ⋯ | ⋯ | ⋯ | — | ⋯ | ? |
 | AMD GPU | `amd_vllm_rocm_6c18cd8f` | vLLM-ROCm | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ? |
+| Huawei Ascend NPU | `ascend_vllm_ascend_b1957851` | vllm-ascend | ✓ | ✓ | ✓ | ✓ | ✓ | — | — | ? |
 | Huawei Ascend NPU | `ascend_vllm_ascend_d4aa9fda` | vllm-ascend | ✓ | ✓ | ✓ | ✓ | ✓ | — | — | ? |
 | Apple Silicon | `apple_mlx_lm_9546b8b5` | mlx-lm | ⋯ | — | — | ⋯ | — | ⋯ | — | ? |
 | Google TPU | `google_vllm_tpu_68cc9ffa` | vllm-tpu | ✓ | — | — | ✓ | — | ✓ | — | ? |

configs/runner_configs/runner_ascend_vllm_ascend_b1957851.yaml.example

@@ -0,0 +1,102 @@
+# AccelMark runner config — ascend_vllm_ascend_b1957851 (vllm-ascend on Huawei Ascend)
+#
+# Copy this file to runner_ascend_vllm_ascend_b1957851.yaml (remove .example suffix)
+# and edit as needed for your hardware. The actual .yaml is gitignored.
+#
+# These settings adapt the runner to your hardware environment.
+# They are recorded in result.json task.extra_config for transparency
+# but are NOT part of the benchmark identity (not hashed into run_id).
+#
+# Merge priority: CLI flags > suite-specific > global defaults > runner defaults
+
+# ── Global defaults (apply to all suites) ─────────────────────────────────────
+
+# Tensor parallel size — number of NPU dies to use for tensor parallelism (default: 1).
+# For Ascend 910C (dual-die): each physical card contains 2 dies.
+# When data_parallel_size > 1, the runner launches one independent engine per die.
+# TP=1 + DP=2 = full single-card throughput with near-linear scaling.
+# For large models (Suite B, 70B) that don't fit per-die, set
+# tensor_parallel_size to the total die count (e.g. 8 for 4×910C cards).
+tensor_parallel_size: 1
+
+# Data parallel size — number of independent engine replicas (default: auto).
+# Auto-detection: if the NPU reports Chip Count >= 2 in board info, dp_size = 2.
+# DP benefits suites where the model fits comfortably on a single die (Suites A, C, H).
+# The following suites override this to dp=1 (see suite-specific section below):
+#   Suite B, G — model too large for single die, must use TP across cards
+#   Suite D    — long-context KV cache makes per-die memory tight, OOM risk
+#   Suite E    — multi-card scaling benchmark, DP would break cross-platform comparability
+#   Suite F    — 0.5B model, engine overhead > throughput gain from parallelism
+# Set explicitly to override: 1 to disable DP, or a higher value for testing.
+# data_parallel_size: 1    # Uncomment to force single-die mode globally
+
+# Disable CANN graph compilation. Set to true if you encounter graph capture
+# errors (e.g. unsupported ops on your CANN version). Equivalent to
+# --enforce-eager on the CLI.
+enforce_eager: false
+
+# Maximum number of sequences in a batch — per-engine value (default: 512).
+# Reduce on lower-memory NPUs: 256 for 32 GB, 128 for 16 GB or less.
+# When data_parallel_size > 1, this value is automatically divided by dp_size
+# (e.g. 512 → 256 per worker for DP=2) to avoid shared-memory broadcast
+# contention between engines on the same physical card.
+max_num_seqs: 512
+
+# Fraction of NPU HBM reserved for the KV cache (default: 0.90).
+# Reduce if you get OOM errors: try 0.85 or 0.80 for tighter memory budgets.
+gpu_memory_utilization: 0.90
+
+# Pass-through kwargs forwarded directly to vLLM LLM() / AsyncEngineArgs().
+# Use for any vLLM setting not listed above. See vLLM docs for valid keys:
+# https://docs.vllm.ai/en/latest/api/vllm/engine/arg_utils.html
+# engine_kwargs:
+#   swap_space: 8
+#   max_seq_len_to_capture: 4096
+
+# ── Suite-specific overrides ───────────────────────────────────────────────────
+# Keys here override the global defaults above for a specific suite only.
+# Only the section matching the current suite is used — other suite sections
+# are never loaded or recorded.
+
+suites:
+  # Suite A: Llama-3-8B — DP=2 benefits greatly (~2× throughput, near-linear)
+  #   dp auto-detected (910C → 2), no override needed.
+
+  # Suite B: Llama-3-70B — model too large for single die, TP across cards.
+  #   DP auto-detected but model won't fit → tp_size forces multi-card.
+  #   dp is already 1 in practice because available_dies < dies_per_card * tp_size.
+  #   Explicit override for clarity:
+  suite_B:
+    data_parallel_size: 1
+
+  # Suite C: Llama-3.1-8B quantization benchmark — DP=2 benefits.
+  #   dp auto-detected, no override needed.
+
+  suite_D:
+    # Long-context suite (~28K input tokens) — tighter memory budget.
+    # Per-die KV cache is large; DP=2 risks OOM at higher concurrency.
+    max_num_seqs: 64
+    gpu_memory_utilization: 0.85
+    data_parallel_size: 1
+
+  # Suite E: Multi-card scaling benchmark (1x/2x/4x cards).
+  #   DP would inflate chip_count with intra-card parallelism,
+  #   breaking cross-platform scaling comparisons. Force single-engine.
+  suite_E:
+    data_parallel_size: 1
+
+  suite_F:
+    # Consumer/edge suite (Qwen2.5-0.5B) — single-die only.
+    # 0.5B model = ~1GB, engine overhead far exceeds any DP gain.
+    # User testing confirms: single die performs better than DP=2.
+    max_num_seqs: 128
+    data_parallel_size: 1
+
+  # Suite G: Mixtral-8x7B MoE — model too large for single die (like Suite B).
+  #   dp auto-falls-back to 1 because model won't fit.
+  suite_G:
+    data_parallel_size: 1
+
+  # Suite H: Mistral-7B SWA — DP=2 benefits.
+  #   7B model fits on single die, SWA caps KV cache at window=4096.
+  #   dp auto-detected, no override needed.

configs/runner_configs/runner_ascend_vllm_ascend_d4aa9fda.yaml.example

@@ -0,0 +1,53 @@
+# AccelMark runner config — ascend_vllm_ascend_d4aa9fda (vllm-ascend on Huawei Ascend)
+#
+# Copy this file to runner_ascend_vllm_ascend_d4aa9fda.yaml (remove .example suffix)
+# and edit as needed for your hardware. The actual .yaml is gitignored.
+#
+# These settings adapt the runner to your hardware environment.
+# They are recorded in result.json task.extra_config for transparency
+# but are NOT part of the benchmark identity (not hashed into run_id).
+#
+# Merge priority: CLI flags > suite-specific > global defaults > runner defaults
+
+# ── Global defaults (apply to all suites) ─────────────────────────────────────
+
+# Tensor parallel size — number of NPUs to use (default: 1).
+# IMPORTANT for Ascend 910C: 910C is a dual-die package treated as one physical
+# card. TP=1 exercises only one die (~50% of card throughput). For results that
+# reflect full single-card capability, set tensor_parallel_size: 2.
+tensor_parallel_size: 1
+
+# Disable CANN graph compilation. Set to true if you encounter graph capture
+# errors (e.g. unsupported ops on your CANN version). Equivalent to
+# --enforce-eager on the CLI.
+enforce_eager: false
+
+# Maximum number of sequences in a batch (default: 512).
+# Reduce on lower-memory NPUs: 256 for 32 GB, 128 for 16 GB or less.
+max_num_seqs: 512
+
+# Fraction of NPU HBM reserved for the KV cache (default: 0.90).
+# Reduce if you get OOM errors: try 0.85 or 0.80 for tighter memory budgets.
+gpu_memory_utilization: 0.90
+
+# Pass-through kwargs forwarded directly to vLLM LLM() / AsyncEngineArgs().
+# Use for any vLLM setting not listed above. See vLLM docs for valid keys:
+# https://docs.vllm.ai/en/latest/api/vllm/engine/arg_utils.html
+# engine_kwargs:
+#   swap_space: 8
+#   max_seq_len_to_capture: 4096
+
+# ── Suite-specific overrides ───────────────────────────────────────────────────
+# Keys here override the global defaults above for a specific suite only.
+# Only the section matching the current suite is used — other suite sections
+# are never loaded or recorded.
+
+suites:
+  suite_D:
+    # Long-context suite (~28K input tokens) — tighter memory budget.
+    max_num_seqs: 64
+    gpu_memory_utilization: 0.85
+
+  suite_F:
+    # Consumer/edge suite (Qwen2.5-0.5B) — single-chip only, lower concurrency.
+    max_num_seqs: 128

leaderboard/serve.py

@@ -0,0 +1,25 @@
+import http.server
+import socketserver
+import mimetypes
+
+# Force correct MIME types for JavaScript modules
+mimetypes.init()
+mimetypes.add_type('application/javascript', '.js')
+mimetypes.add_type('application/javascript', '.mjs')
+mimetypes.add_type('text/css', '.css')
+mimetypes.add_type('image/svg+xml', '.svg')
+mimetypes.add_type('application/json', '.json')
+
+PORT = 8000
+
+class Handler(http.server.SimpleHTTPRequestHandler):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, directory='site', **kwargs)
+
+    def guess_type(self, path):
+        mime, _ = mimetypes.guess_type(path)
+        return mime or 'application/octet-stream'
+
+with socketserver.TCPServer(("", PORT), Handler) as httpd:
+    print(f"Serving at http://localhost:{PORT}")
+    httpd.serve_forever()

leaderboard/site/assets/css/components.css

@@ -185,16 +185,14 @@
   font-weight: 500;
 }
 
-/* Hero — centered text stack, no rounded clip. Background lives on
-   body::before so the gradient bleeds into the page naturally. */
+/* Hero */
 .hero {
-  padding: 3.25rem 1rem 2.75rem;
+  padding: 4rem 1.5rem 3rem;
   margin-bottom: 1.5rem;
-  text-align: center;
 }
 .hero h1 {
-  margin: 0 auto 0.55rem;
-  font-size: 3rem;
+  margin: 0 0 0.55rem;
+  font-size: 2.75rem;
   font-weight: 700;
   color: var(--fg-strong);
   letter-spacing: -0.025em;
@@ -206,20 +204,90 @@
   white-space: nowrap;
 }
 .hero .hero-sub {
-  margin: 0 auto 1.4rem;
-  font-size: 1.15rem;
-  color: var(--fg-strong);
+  margin: 0 0 0;
+  font-size: 1.05rem;
+  color: var(--fg-muted);
   font-weight: 400;
   letter-spacing: -0.005em;
-  line-height: 1.35;
-  white-space: nowrap;
+  line-height: 1.45;
 }
-.hero .tagline {
-  color: var(--fg-muted);
-  margin: 0 auto 1.9rem;
-  font-size: 1rem;
-  line-height: 1.6;
-  max-width: 54ch;
+
+/* Hero: side-by-side text + comparison chart */
+.hero-main {
+  display: flex; align-items: center; justify-content: center;
+  gap: 2rem; margin: 0 auto 1.6rem;
+}
+.hero-text { flex: 0 0 auto; }
+.hero-text h1 { max-width: none; }
+.hero-text .hero-sub { max-width: none; white-space: nowrap; }
+
+/* Chart card */
+.hero-chart-card {
+  flex-shrink: 0;
+  background: var(--bg-elev);
+  border: 1px solid var(--border-soft);
+  border-radius: var(--r-md, 8px);
+  padding: 1rem 1.15rem 0.9rem;
+}
+.hero-chart-eyebrow {
+  display: block; font-size: 0.58rem; text-transform: uppercase;
+  letter-spacing: 0.1em; color: var(--fg-muted); margin-bottom: 0.5rem;
+  text-align: center; font-weight: 600;
+}
+.hero-chart-row {
+  display: flex; gap: 0.8rem; align-items: flex-end;
+}
+.hero-chart-panel { text-align: center; }
+.hero-chart-panel-title {
+  display: block; font-size: 0.62rem; text-transform: uppercase;
+  letter-spacing: 0.06em; color: var(--fg-muted); margin-bottom: 0.3rem;
+  font-weight: 600;
+}
+.hero-chart-panel-note {
+  display: block; font-size: 0.7rem; margin-top: 0.3rem;
+  font-weight: 600; line-height: 1.3;
+}
+.hero-chart-panel-note.advantage {
+  color: #16a34a;
+}
+.hero-chart-panel-note.disadvantage {
+  color: #dc2626;
+}
+.winner-chip {
+  display: inline-block;
+  padding: 0.05rem 0.4rem;
+  background: color-mix(in srgb, var(--accent-2) 14%, transparent);
+  color: var(--accent-2);
+  border-radius: 3px;
+  font-weight: 600;
+  font-size: 0.68rem;
+  letter-spacing: 0.01em;
+}
+.hero-chart-caption {
+  font-size: 0.7rem; color: var(--fg-muted); margin-top: 0.65rem;
+  text-align: center; font-weight: 400; line-height: 1.5; max-width: 42ch;
+  margin-left: auto; margin-right: auto;
+}
+
+@media (max-width: 840px) {
+  .hero-main { flex-direction: column; gap: 1.5rem; }
+  .hero-text { text-align: center; }
+  .hero-text .hero-sub { margin-left: auto; margin-right: auto; max-width: 44ch; white-space: normal; }
+}
+@media (max-width: 560px) {
+  .hero { padding: 2.5rem 1rem 2rem; }
+  .hero h1 { font-size: 2rem; }
+  .hero-chart-row { gap: 0.3rem; }
+  .hero-chart-panel-title { font-size: 0.56rem; }
+}
+
+.hero-reversal-hint {
+  color: var(--accent);
+  margin: 0 auto 1.5rem;
+  font-size: 0.92rem;
+  font-weight: 500;
+  max-width: 50ch;
+  opacity: 0.9;
 }
 
 .hero-stats {

leaderboard/site/assets/css/home.css

@@ -186,6 +186,11 @@
 .suite-card .lb-row-chip .lb-row-fw .fw-ver {
   font-size: 0.88em;
 }
+.suite-card .lb-row-chip .lb-row-fw .fw-ops {
+  font-size: 0.82em;
+  color: var(--accent);
+  font-weight: 500;
+}
 .suite-card .lb-row-sub {
   font-size: 0.68rem;
   line-height: 1.2;
@@ -324,6 +329,11 @@
   font-size: 0.85em;
   color: var(--fg-faint);
 }
+.lb-row-chip .lb-row-fw .fw-ops {
+  font-size: 0.82em;
+  color: var(--accent);
+  font-weight: 500;
+}
 .lb-row-name:hover {
   color: var(--accent-2);
   text-decoration: underline;

leaderboard/site/assets/css/layout.css

@@ -108,7 +108,7 @@ footer a:hover { color: var(--fg); border-bottom-color: var(--fg); text-decorati
   color: var(--fg-muted);
   font-size: 0.92rem;
   line-height: 1.45;
-  text-align: right;
+  text-align: left;
   flex: 0 1 auto;
 }
 

leaderboard/site/assets/css/pages.css

@@ -2,8 +2,8 @@
 
 .page-hero {
   text-align: center;
-  padding: 2rem 1rem 1.5rem;
-  max-width: 48rem;
+  padding: 2rem 2rem 1.5rem;
+  max-width: 72rem;
   margin: 0 auto;
 }
 .page-hero .eyebrow { display: block; margin-bottom: 0.5rem; }
@@ -19,7 +19,7 @@
   color: var(--fg-muted);
   font-size: 1rem;
   line-height: 1.6;
-  max-width: 42rem;
+  max-width: none;
   margin: 0 auto 1.25rem;
 }
 .hero-stats.compact { margin-bottom: 1rem; padding: 0.75rem 0; }