ci: Add CodSpeed continuous benchmarking

.github/workflows/codspeed.yml

@@ -0,0 +1,71 @@
+# evmone: Fast Ethereum Virtual Machine implementation
+# Copyright 2026 The evmone Authors.
+# SPDX-License-Identifier: Apache-2.0
+
+name: CodSpeed
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+  # `workflow_dispatch` allows CodSpeed to trigger backtest
+  # performance analysis in order to generate initial data.
+  workflow_dispatch:
+
+env:
+  BUILD_DIR: build
+  CMAKE_BUILD_PARALLEL_LEVEL: 4
+  # The newest GCC on the ubuntu-24.04 image, which otherwise defaults to GCC 13.
+  CC: gcc-14
+  CXX: g++-14
+
+jobs:
+  benchmark:
+    name: Benchmark
+    # CodSpeed ships its instrumented Valgrind for Ubuntu 22.04, Ubuntu 24.04 and Debian 12 only,
+    # and bails out with "Unsupported system" anywhere else, so the image cannot be `latest`.
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read # required for actions/checkout
+      id-token: write # required for OIDC authentication with CodSpeed
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          submodules: recursive
+
+      - name: Install GMP
+        run: sudo apt-get -q update && sudo apt-get -qy install libgmp-dev
+
+      - name: Configure
+        # The Release build type keeps the -O3 optimizations on, -g adds the debug information
+        # needed by CodSpeed to attribute the measurements to the source code. CodSpeed recommends
+        # (and warns about) RelWithDebInfo, but that would downgrade the optimizations to -O2.
+        run: >
+          cmake -S . -B $BUILD_DIR
+          -DCMAKE_BUILD_TYPE=Release
+          -DCMAKE_C_FLAGS=-g
+          -DCMAKE_CXX_FLAGS=-g
+          -DBUILD_SHARED_LIBS=OFF
+          -DEVMONE_TESTING=ON
+          -DEVMONE_PRECOMPILES_GMP=ON
+          -DEVMONE_PRECOMPILES_LIBSECP256K1=ON
+          -DCODSPEED_MODE=simulation
+
+      - name: Build
+        run: cmake --build $BUILD_DIR --target evmone-bench evmone-precompiles-bench
+
+      # CodSpeed aggregates the results of all the runs of a single workflow run.
+      - name: Benchmark the baseline VM
+        uses: CodSpeedHQ/action@v5
+        with:
+          mode: simulation
+          run: ${{ env.BUILD_DIR }}/bin/evmone-bench test/evm-benchmarks/benchmarks --benchmark_filter='baseline/(analyse|execute)'
+
+      # Both the mainnet inputs and the modexp parameter sweep: the mainnet moduli are all odd
+      # and 256-bit, so they cover a single code path and hide the work done on the others.
+      - name: Benchmark the precompiles
+        uses: CodSpeedHQ/action@v5
+        with:
+          mode: simulation
+          run: ${{ env.BUILD_DIR }}/bin/evmone-precompiles-bench

README.md

@@ -3,6 +3,7 @@
 [![ethereum badge]][ethereum]
 [![readme style standard badge]][standard readme]
 [![codecov badge]][codecov]
+[![codspeed badge]][codspeed]
 [![circleci badge]][circleci]
 [![appveyor badge]][appveyor]
 [![license badge]][Apache License, Version 2.0]
@@ -128,6 +129,7 @@ Licensed under the [Apache License, Version 2.0].
 [appveyor]: https://ci.appveyor.com/project/chfast/evmone/branch/master
 [circleci]: https://circleci.com/gh/ethereum/evmone/tree/master
 [codecov]: https://codecov.io/gh/ethereum/evmone/
+[codspeed]: https://app.codspeed.io/ipsilon/evmone
 [Apache License, Version 2.0]: LICENSE
 [ethereum]: https://ethereum.org
 [EVMC]: https://github.com/ethereum/evmc
@@ -143,6 +145,7 @@ Licensed under the [Apache License, Version 2.0].
 [appveyor badge]: https://img.shields.io/appveyor/ci/chfast/evmone/master.svg?logo=appveyor
 [circleci badge]: https://img.shields.io/circleci/project/github/ethereum/evmone/master.svg?logo=circleci
 [codecov badge]: https://img.shields.io/codecov/c/github/ethereum/evmone.svg?logo=codecov
+[codspeed badge]: https://img.shields.io/endpoint?url=https://codspeed.io/badge.json
 [ethereum badge]: https://img.shields.io/badge/ethereum-EVM-informational.svg?logo=ethereum
 [license badge]: https://img.shields.io/github/license/ethereum/evmone.svg?logo=apache
 [readme style standard badge]: https://img.shields.io/badge/readme%20style-standard-brightgreen.svg

test/CMakeLists.txt

@@ -13,20 +13,38 @@ find_package(CLI11 CONFIG REQUIRED)
 hunter_add_package(GTest)
 find_package(GTest CONFIG REQUIRED)
 
-FetchContent_Declare(
-    benchmark
-    URL https://github.com/google/benchmark/archive/refs/tags/v1.9.5.tar.gz
-    URL_HASH SHA256=9631341c82bac4a288bef951f8b26b41f69021794184ece969f8473977eaa340
-)
-block()
+block(PROPAGATE benchmark_libs)
     set(BUILD_SHARED_LIBS OFF)
     set(BENCHMARK_ENABLE_TESTING OFF)
     set(BENCHMARK_ENABLE_INSTALL OFF)
+    set(benchmark_libs benchmark benchmark_main)
+    if(NOT CODSPEED_MODE)
+        FetchContent_Declare(
+            benchmark
+            URL https://github.com/google/benchmark/archive/refs/tags/v1.9.5.tar.gz
+            URL_HASH SHA256=9631341c82bac4a288bef951f8b26b41f69021794184ece969f8473977eaa340
+        )
+    else()
+        # Replace google/benchmark with CodSpeed's fork of it to report the benchmark results
+        # to https://codspeed.io. It provides the same benchmark::benchmark target,
+        # so the benchmark targets need no changes.
+        FetchContent_Declare(
+            benchmark
+            # Fork of v2.4.0 with fixes not yet available upstream.
+            GIT_REPOSITORY https://github.com/chfast/codspeed-cpp
+            GIT_TAG 0ecdd6e7d8f9e858e8e0b2fbe54ff7f427bec528
+            GIT_SHALLOW FALSE
+            SOURCE_SUBDIR google_benchmark
+        )
+        list(APPEND benchmark_libs codspeed instrument_hooks)
+    endif()
     FetchContent_MakeAvailable(benchmark)
 endblock()
+
 set_target_properties(
-    benchmark benchmark_main PROPERTIES
-    COMPILE_OPTIONS $<$<CXX_COMPILER_ID:GNU,Clang,AppleClang>:-w>
+    ${benchmark_libs} PROPERTIES
+    COMPILE_OPTIONS "$<$<COMPILE_LANG_AND_ID:C,GNU,Clang,AppleClang>:-w>;$<$<COMPILE_LANG_AND_ID:CXX,GNU,Clang,AppleClang>:-w>"
+    C_CLANG_TIDY ""
     CXX_CLANG_TIDY ""
     SYSTEM TRUE  # TODO(cmake-3.26): Use the SYSTEM option of FetchContent_Declare().
 )

test/bench/bench.cpp

@@ -116,15 +116,15 @@ void register_benchmarks(std::span<const BenchmarkCase> benchmark_cases)
     {
         if (advanced_vm != nullptr)
         {
-            RegisterBenchmark("advanced/analyse/" + b.name, [&b](State& state) {
+            RegisterBenchmark(bench_name("advanced/analyse/" + b.name), [&b](State& state) {
                 bench_analyse<advanced::AdvancedCodeAnalysis, advanced_analyse>(
                     state, default_revision, b.code);
             })->Unit(kMicrosecond);
         }
 
         if (baseline_vm != nullptr)
         {
-            RegisterBenchmark("baseline/analyse/" + b.name, [&b](State& state) {
+            RegisterBenchmark(bench_name("baseline/analyse/" + b.name), [&b](State& state) {
                 bench_analyse<baseline::CodeAnalysis, baseline_analyse>(
                     state, default_revision, b.code);
             })->Unit(kMicrosecond);
@@ -136,31 +136,31 @@ void register_benchmarks(std::span<const BenchmarkCase> benchmark_cases)
 
             if (advanced_vm != nullptr)
             {
-                const auto name = "advanced/execute/" + case_name;
+                const auto name = bench_name("advanced/execute/" + case_name);
                 RegisterBenchmark(name, [&vm = *advanced_vm, &b, &input](State& state) {
                     bench_advanced_execute(state, vm, b.code, input.input, input.expected_output);
                 })->Unit(kMicrosecond);
             }
 
             if (baseline_vm != nullptr)
             {
-                const auto name = "baseline/execute/" + case_name;
+                const auto name = bench_name("baseline/execute/" + case_name);
                 RegisterBenchmark(name, [&vm = *baseline_vm, &b, &input](State& state) {
                     bench_baseline_execute(state, vm, b.code, input.input, input.expected_output);
                 })->Unit(kMicrosecond);
             }
 
             if (basel_cg_vm != nullptr)
             {
-                const auto name = "bnocgoto/execute/" + case_name;
+                const auto name = bench_name("bnocgoto/execute/" + case_name);
                 RegisterBenchmark(name, [&vm = *basel_cg_vm, &b, &input](State& state) {
                     bench_baseline_execute(state, vm, b.code, input.input, input.expected_output);
                 })->Unit(kMicrosecond);
             }
 
             for (auto& [vm_name, vm] : registered_vms)
             {
-                const auto name = std::string{vm_name} + "/total/" + case_name;
+                const auto name = bench_name(std::string{vm_name} + "/total/" + case_name);
                 RegisterBenchmark(name, [&vm, &b, &input](State& state) {
                     bench_evmc_execute(state, vm, b.code, input.input, input.expected_output);
                 })->Unit(kMicrosecond);
@@ -234,7 +234,10 @@ std::tuple<int, std::vector<BenchmarkCase>> parseargs(int argc, char** argv)
 
 int main(int argc, char** argv)
 {
+#ifndef CODSPEED_ENABLED
+    // Not available in the google/benchmark version CodSpeed's fork is based on, nor needed there.
     MaybeReenterWithoutASLR(argc, argv);
+#endif
 
     using namespace evmone::test;
     try

test/bench/helpers.hpp

@@ -11,11 +11,26 @@
 #include <evmone/advanced_execution.hpp>
 #include <evmone/baseline.hpp>
 #include <evmone/vm.hpp>
+#include <source_location>
 
 namespace evmone::test
 {
 extern std::map<std::string_view, evmc::VM> registered_vms;
 
+/// Decorates a dynamically registered benchmark name with the location of the registration.
+/// CodSpeed identifies benchmarks by the "source_file::name" URI, which the BENCHMARK() macro
+/// adds automatically, but RegisterBenchmark() does not. Without CodSpeed this is a no-op.
+/// The default argument is evaluated at the call site, naming the file registering the benchmark.
+inline std::string bench_name(std::string name,
+    [[maybe_unused]] const std::source_location loc = std::source_location::current())
+{
+#ifdef CODSPEED_ENABLED
+    return codspeed::get_path_relative_to_workspace(loc.file_name()) + "::" + std::move(name);
+#else
+    return name;
+#endif
+}
+
 constexpr auto default_revision = EVMC_ISTANBUL;
 constexpr auto default_gas_limit = std::numeric_limits<int64_t>::max();
 

test/bench/synthetic_benchmarks.cpp

@@ -252,17 +252,18 @@ void register_synthetic_benchmarks()
 
     for (auto& [vm_name, vm] : registered_vms)
     {
-        RegisterBenchmark(std::string{vm_name} + "/total/synth/loop_v1",
+        RegisterBenchmark(bench_name(std::string{vm_name} + "/total/synth/loop_v1"),
             [&vm](State& state) { bench_evmc_execute(state, vm, generate_loop_v1({})); });
-        RegisterBenchmark(std::string{vm_name} + "/total/synth/loop_v2",
+        RegisterBenchmark(bench_name(std::string{vm_name} + "/total/synth/loop_v2"),
             [&vm](State& state) { bench_evmc_execute(state, vm, generate_loop_v2({})); });
     }
 
     for (const auto params : params_list)
     {
         for (auto& [vm_name, vm] : registered_vms)
         {
-            RegisterBenchmark(std::string{vm_name} + "/total/synth/" + to_string(params),
+            RegisterBenchmark(
+                bench_name(std::string{vm_name} + "/total/synth/" + to_string(params)),
                 [&vm, params](
                     State& state) { bench_evmc_execute(state, vm, generate_code(params)); })
                 ->Unit(kMicrosecond);