Add generated-Ok() code-size + execution-speed benchmark harness

.github/workflows/code-size.yml

@@ -0,0 +1,175 @@
+name: Code size
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    # Only run when something that can change generated code size moves.
+    paths:
+      - 'compiler/**'
+      - 'runtime/**'
+      - 'embossc'
+      - 'testdata/benchmark.emb'
+      - 'testdata/many_conditionals.emb'
+      - 'scripts/size_bench.py'
+      - 'scripts/size_comment.py'
+      - 'scripts/emboss_insncount.c'
+      - 'scripts/build_qemu_plugin.sh'
+      - '.github/workflows/code-size.yml'
+
+# Read the PR's commits; post/update one sticky size comment.
+permissions:
+  contents: read
+  pull-requests: write
+
+# Supersede an in-flight size run when the PR is pushed again.
+concurrency:
+  group: code-size-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  size-report:
+    name: Generated code size
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out PR head (full history for merge-base + revision checkouts)
+        uses: actions/checkout@v6
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 0
+
+      # embossc formats generated headers with clang-format (the clang-format
+      # PyPI package, pinned in requirements.txt), so the codegen run needs the
+      # Emboss Python deps. setup-python avoids the runner's externally-managed
+      # system Python (PEP 668).
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+          cache: 'pip'
+          cache-dependency-path: requirements.txt
+      - name: Install Emboss Python deps (clang-format for codegen)
+        run: pip install -r requirements.txt
+
+      - name: Install compilers and speed-benchmark build deps
+        # clang + arm-none-eabi (Cortex-M4 size); arm-linux-gnueabihf + qemu build
+        # deps power the opt-in execution-speed run (see the qemu step below).
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y clang gcc-arm-none-eabi \
+            g++-arm-linux-gnueabihf \
+            meson ninja-build pkg-config libglib2.0-dev
+
+      # embedded code-size targets compile against the MicroBlaze gcc toolchain
+      # hard-coded under /opt/microblaze/...; cache the Bootlin tarball in a
+      # runner-writable path (caching /opt fights root perms on restore), then
+      # extract it into place. clang has no MicroBlaze back end, so MicroBlaze is
+      # gcc-only.
+      - name: Cache MicroBlaze (Bootlin) toolchain tarball
+        uses: actions/cache@v4
+        with:
+          path: ~/mb-toolchain/mb.tar.xz
+          key: bootlin-microblazebe--glibc--stable-2025.08-1
+      - name: Provision MicroBlaze toolchain
+        run: |
+          set -euo pipefail
+          url="https://toolchains.bootlin.com/downloads/releases/toolchains/microblazebe/tarballs/microblazebe--glibc--stable-2025.08-1.tar.xz"
+          if [ ! -f "$HOME/mb-toolchain/mb.tar.xz" ]; then
+            mkdir -p "$HOME/mb-toolchain"
+            curl -fsSL "$url" -o "$HOME/mb-toolchain/mb.tar.xz"
+          fi
+          sudo mkdir -p /opt/microblaze
+          sudo tar -C /opt/microblaze -xJf "$HOME/mb-toolchain/mb.tar.xz"
+          test -x /opt/microblaze/microblazebe--glibc--stable-2025.08-1/bin/microblaze-buildroot-linux-gnu-g++
+
+      # Little-endian MicroBlaze. The *authoritative* LE size target is the
+      # AMD/Xilinx Vitis toolchain, but that is proprietary and locally-licensed,
+      # so it can't run in public CI -- size_bench.py's have() skips it here. The
+      # open-source Bootlin microblazeel build is the CI LE proxy (and provides
+      # the LE speed binary below); locally, both are measured.
+      - name: Cache MicroBlaze LE (Bootlin) toolchain tarball
+        uses: actions/cache@v4
+        with:
+          path: ~/mb-toolchain/mbel.tar.xz
+          key: bootlin-microblazeel--glibc--stable-2025.08-1
+      - name: Provision MicroBlaze LE toolchain
+        run: |
+          set -euo pipefail
+          url="https://toolchains.bootlin.com/downloads/releases/toolchains/microblazeel/tarballs/microblazeel--glibc--stable-2025.08-1.tar.xz"
+          if [ ! -f "$HOME/mb-toolchain/mbel.tar.xz" ]; then
+            mkdir -p "$HOME/mb-toolchain"
+            curl -fsSL "$url" -o "$HOME/mb-toolchain/mbel.tar.xz"
+          fi
+          sudo mkdir -p /opt/microblaze
+          sudo tar -C /opt/microblaze -xJf "$HOME/mb-toolchain/mbel.tar.xz"
+          test -x /opt/microblaze/microblazeel--glibc--stable-2025.08-1/bin/microblazeel-buildroot-linux-gnu-g++
+
+      # Execution-speed metric (retired-instruction count) needs a plugin-enabled
+      # qemu-user, which the distro build is not -- so build a minimal one from
+      # source (cached; ~a few minutes on a cache miss, seconds otherwise). This
+      # is best-effort: if it fails, the speed run degrades to host wall-clock and
+      # the size report is unaffected.
+      - name: Cache plugin-enabled qemu build
+        uses: actions/cache@v4
+        with:
+          path: ~/emboss-qemu
+          key: emboss-qemu-10.0.11-${{ hashFiles('scripts/emboss_insncount.c', 'scripts/build_qemu_plugin.sh') }}
+      - name: Build plugin-enabled qemu + instruction-count plugin
+        continue-on-error: true
+        run: |
+          if [ ! -x "$HOME/emboss-qemu/qemu-build/qemu-microblazeel" ]; then
+            scripts/build_qemu_plugin.sh 10.0.11 "$HOME/emboss-qemu"
+          fi
+
+      - name: Measure generated code size (merge-base vs PR head)
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          set -euo pipefail
+          # Baseline against the merge-base, NOT the base-branch tip, so codegen
+          # that landed on the base after this PR branched is not misattributed.
+          # size_bench.py holds the benchmark schema fixed (pulled forward from
+          # head), so only the generator/runtime under test differs.
+          base="$(git merge-base "$BASE_SHA" "$HEAD_SHA")"
+          echo "Comparing merge-base $base  ->  head $HEAD_SHA"
+          # Wire the plugin-enabled qemu for retired-instruction counts when the
+          # build above succeeded; otherwise --speed degrades to wall-clock only.
+          if [ -x "$HOME/emboss-qemu/qemu-build/qemu-microblazeel" ] \
+             && [ -f "$HOME/emboss-qemu/emboss_insncount.so" ]; then
+            export EMBOSS_QEMU_DIR="$HOME/emboss-qemu/qemu-build"
+            export EMBOSS_QEMU_PLUGIN="$HOME/emboss-qemu/emboss_insncount.so"
+          fi
+          python3 scripts/size_bench.py --revisions "$base" "$HEAD_SHA" --speed --out-dir "$RUNNER_TEMP/size"
+          # Fail loudly rather than post an empty table if the head build produced
+          # no data (missing toolchain/dep or a codegen break).
+          python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); h=d["revisions"][-1]["results"]; sys.exit(0 if any(cfg.get("benchmark") for t in h.values() for c in t.values() for cfg in c.values()) else 1)' "$RUNNER_TEMP/size/size_bench.json" \
+            || { echo "::error::size_bench produced no head benchmark data (toolchain/codegen failure); see output above."; exit 1; }
+          python3 scripts/size_comment.py "$RUNNER_TEMP/size/size_bench.json" > "$RUNNER_TEMP/size/comment.md"
+
+      - name: Post / update size comment
+        # Fork PRs get a read-only GITHUB_TOKEN; same-repo PRs (the chain branches)
+        # can comment. (Fork-safe upgrade: a separate workflow_run job.)
+        if: ${{ github.event.pull_request.head.repo.full_name == github.repository }}
+        uses: actions/github-script@v7
+        env:
+          COMMENT_PATH: ${{ runner.temp }}/size/comment.md
+        with:
+          script: |
+            const fs = require('fs');
+            const marker = '<!-- emboss-size-bench -->';
+            let body;
+            try {
+              body = fs.readFileSync(process.env.COMMENT_PATH, 'utf8');
+            } catch (e) {
+              return;  // measure step failed; its red check is the signal.
+            }
+            const { owner, repo } = context.repo;
+            const issue_number = context.issue.number;
+            const comments = await github.paginate(github.rest.issues.listComments, {
+              owner, repo, issue_number, per_page: 100,
+            });
+            const existing = comments.find(c => c.body && c.body.includes(marker));
+            if (existing) {
+              await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body });
+            } else {
+              await github.rest.issues.createComment({ owner, repo, issue_number, body });
+            }

scripts/build_qemu_plugin.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Builds what `size_bench.py --speed` needs for deterministic retired-instruction
+# counts: a plugin-enabled qemu-user (arm + microblaze + microblazeel) and the
+# instruction-counter TCG plugin (scripts/emboss_insncount.c). The distro
+# qemu-user is built without --enable-plugins, so a from-source qemu is required.
+#
+# Usage:
+#   scripts/build_qemu_plugin.sh [QEMU_VERSION] [DEST_DIR]
+# then point the harness at the results:
+#   export EMBOSS_QEMU_DIR=<DEST_DIR>/qemu-build
+#   export EMBOSS_QEMU_PLUGIN=<DEST_DIR>/emboss_insncount.so
+#   python3 scripts/size_bench.py --speed --out-dir out
+#
+# Build deps (Debian/Ubuntu): meson ninja-build pkg-config libglib2.0-dev python3.
+# Speed toolchains (used by size_bench.py, not by this script): the Bootlin
+# microblaze[el] gcc and g++-arm-linux-gnueabihf. The distro qemu-user package is
+# not needed -- the from-source binaries built here are invoked directly.
+set -euo pipefail
+
+QEMU_VERSION="${1:-10.0.11}"
+DEST="${2:-$HOME/emboss-qemu}"
+SCRIPTS="$(cd "$(dirname "$0")" && pwd)"
+
+mkdir -p "$DEST"
+cd "$DEST"
+
+TARBALL="qemu-${QEMU_VERSION}.tar.xz"
+[ -f "$TARBALL" ] || curl -fsSL "https://download.qemu.org/${TARBALL}" -o "$TARBALL"
+[ -d "qemu-${QEMU_VERSION}" ] || tar -xf "$TARBALL"
+
+echo "Configuring qemu-user (arm, microblaze, microblazeel) with plugins..."
+rm -rf qemu-build && mkdir qemu-build && cd qemu-build
+"../qemu-${QEMU_VERSION}/configure" \
+  --target-list=arm-linux-user,microblaze-linux-user,microblazeel-linux-user \
+  --enable-plugins --disable-system --disable-tools --disable-docs \
+  --disable-werror --disable-debug-info
+ninja qemu-arm qemu-microblaze qemu-microblazeel
+cd ..
+
+echo "Building the instruction-counter plugin..."
+gcc -O2 -shared -fPIC \
+  -I"qemu-${QEMU_VERSION}/include/qemu" $(pkg-config --cflags glib-2.0) \
+  -o "$DEST/emboss_insncount.so" "$SCRIPTS/emboss_insncount.c"
+
+cat <<EOF
+
+Built:
+  qemu:   $DEST/qemu-build/{qemu-arm,qemu-microblaze,qemu-microblazeel}
+  plugin: $DEST/emboss_insncount.so
+
+Now run the speed benchmark with:
+  export EMBOSS_QEMU_DIR=$DEST/qemu-build
+  export EMBOSS_QEMU_PLUGIN=$DEST/emboss_insncount.so
+  python3 scripts/size_bench.py --speed --out-dir out
+EOF

scripts/emboss_insncount.c

@@ -0,0 +1,63 @@
+/* Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * Minimal QEMU TCG plugin used by size_bench.py --speed: counts retired guest
+ * instructions and prints the total to stderr at exit as `insns: <N>`. The count
+ * is deterministic for a given (binary, guest arch), which makes it a stable
+ * cross-architecture execution-speed metric where wall-clock is not.
+ *
+ * It uses an inline per-vCPU add (no per-instruction callback), so emulation runs
+ * at near-native qemu speed even for billions of instructions.
+ *
+ * Built against the QEMU 10.x plugin API (version 4). QEMU's own contrib/plugins
+ * no longer ships an instruction counter (libinsn was dropped), hence this local
+ * plugin. Build it with scripts/build_qemu_plugin.sh.
+ */
+
+#include <inttypes.h>
+#include <stddef.h>
+#include <stdio.h>
+
+#include <glib.h>
+#include <qemu-plugin.h>
+
+QEMU_PLUGIN_EXPORT int qemu_plugin_version = QEMU_PLUGIN_VERSION;
+
+static struct qemu_plugin_scoreboard *score;
+static qemu_plugin_u64 insn_count;
+
+static void vcpu_tb_trans(qemu_plugin_id_t id, struct qemu_plugin_tb *tb) {
+  size_t n = qemu_plugin_tb_n_insns(tb);
+  for (size_t i = 0; i < n; i++) {
+    struct qemu_plugin_insn *insn = qemu_plugin_tb_get_insn(tb, i);
+    qemu_plugin_register_vcpu_insn_exec_inline_per_vcpu(
+        insn, QEMU_PLUGIN_INLINE_ADD_U64, insn_count, 1);
+  }
+}
+
+static void plugin_exit(qemu_plugin_id_t id, void *p) {
+  fprintf(stderr, "insns: %" PRIu64 "\n", qemu_plugin_u64_sum(insn_count));
+}
+
+QEMU_PLUGIN_EXPORT int qemu_plugin_install(qemu_plugin_id_t id,
+                                           const qemu_info_t *info, int argc,
+                                           char **argv) {
+  score = qemu_plugin_scoreboard_new(sizeof(uint64_t));
+  insn_count = qemu_plugin_scoreboard_u64(score);
+  qemu_plugin_register_vcpu_tb_trans_cb(id, vcpu_tb_trans);
+  qemu_plugin_register_atexit_cb(id, plugin_exit, NULL);
+  return 0;
+}