diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml new file mode 100644 index 0000000000..009b448c39 --- /dev/null +++ b/.github/workflows/codspeed.yml @@ -0,0 +1,71 @@ +# evmone: Fast Ethereum Virtual Machine implementation +# Copyright 2026 The evmone Authors. +# SPDX-License-Identifier: Apache-2.0 + +name: CodSpeed + +on: + push: + branches: + - master + pull_request: + # `workflow_dispatch` allows CodSpeed to trigger backtest + # performance analysis in order to generate initial data. + workflow_dispatch: + +env: + BUILD_DIR: build + CMAKE_BUILD_PARALLEL_LEVEL: 4 + # The newest GCC on the ubuntu-24.04 image, which otherwise defaults to GCC 13. + CC: gcc-14 + CXX: g++-14 + +jobs: + benchmark: + name: Benchmark + # CodSpeed ships its instrumented Valgrind for Ubuntu 22.04, Ubuntu 24.04 and Debian 12 only, + # and bails out with "Unsupported system" anywhere else, so the image cannot be `latest`. + runs-on: ubuntu-24.04 + permissions: + contents: read # required for actions/checkout + id-token: write # required for OIDC authentication with CodSpeed + steps: + - uses: actions/checkout@v5 + with: + submodules: recursive + + - name: Install GMP + run: sudo apt-get -q update && sudo apt-get -qy install libgmp-dev + + - name: Configure + # The Release build type keeps the -O3 optimizations on, -g adds the debug information + # needed by CodSpeed to attribute the measurements to the source code. CodSpeed recommends + # (and warns about) RelWithDebInfo, but that would downgrade the optimizations to -O2. + run: > + cmake -S . -B $BUILD_DIR + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_C_FLAGS=-g + -DCMAKE_CXX_FLAGS=-g + -DBUILD_SHARED_LIBS=OFF + -DEVMONE_TESTING=ON + -DEVMONE_PRECOMPILES_GMP=ON + -DEVMONE_PRECOMPILES_LIBSECP256K1=ON + -DCODSPEED_MODE=simulation + + - name: Build + run: cmake --build $BUILD_DIR --target evmone-bench evmone-precompiles-bench + + # CodSpeed aggregates the results of all the runs of a single workflow run. + - name: Benchmark the baseline VM + uses: CodSpeedHQ/action@v5 + with: + mode: simulation + run: ${{ env.BUILD_DIR }}/bin/evmone-bench test/evm-benchmarks/benchmarks --benchmark_filter='baseline/(analyse|execute)' + + # Both the mainnet inputs and the modexp parameter sweep: the mainnet moduli are all odd + # and 256-bit, so they cover a single code path and hide the work done on the others. + - name: Benchmark the precompiles + uses: CodSpeedHQ/action@v5 + with: + mode: simulation + run: ${{ env.BUILD_DIR }}/bin/evmone-precompiles-bench diff --git a/README.md b/README.md index 1ae2f1eeea..59d072eadb 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ [![ethereum badge]][ethereum] [![readme style standard badge]][standard readme] [![codecov badge]][codecov] +[![codspeed badge]][codspeed] [![circleci badge]][circleci] [![appveyor badge]][appveyor] [![license badge]][Apache License, Version 2.0] @@ -128,6 +129,7 @@ Licensed under the [Apache License, Version 2.0]. [appveyor]: https://ci.appveyor.com/project/chfast/evmone/branch/master [circleci]: https://circleci.com/gh/ethereum/evmone/tree/master [codecov]: https://codecov.io/gh/ethereum/evmone/ +[codspeed]: https://app.codspeed.io/ipsilon/evmone [Apache License, Version 2.0]: LICENSE [ethereum]: https://ethereum.org [EVMC]: https://github.com/ethereum/evmc @@ -143,6 +145,7 @@ Licensed under the [Apache License, Version 2.0]. [appveyor badge]: https://img.shields.io/appveyor/ci/chfast/evmone/master.svg?logo=appveyor [circleci badge]: https://img.shields.io/circleci/project/github/ethereum/evmone/master.svg?logo=circleci [codecov badge]: https://img.shields.io/codecov/c/github/ethereum/evmone.svg?logo=codecov +[codspeed badge]: https://img.shields.io/endpoint?url=https://codspeed.io/badge.json [ethereum badge]: https://img.shields.io/badge/ethereum-EVM-informational.svg?logo=ethereum [license badge]: https://img.shields.io/github/license/ethereum/evmone.svg?logo=apache [readme style standard badge]: https://img.shields.io/badge/readme%20style-standard-brightgreen.svg diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7743685105..b8105f8238 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -13,20 +13,38 @@ find_package(CLI11 CONFIG REQUIRED) hunter_add_package(GTest) find_package(GTest CONFIG REQUIRED) -FetchContent_Declare( - benchmark - URL https://github.com/google/benchmark/archive/refs/tags/v1.9.5.tar.gz - URL_HASH SHA256=9631341c82bac4a288bef951f8b26b41f69021794184ece969f8473977eaa340 -) -block() +block(PROPAGATE benchmark_libs) set(BUILD_SHARED_LIBS OFF) set(BENCHMARK_ENABLE_TESTING OFF) set(BENCHMARK_ENABLE_INSTALL OFF) + set(benchmark_libs benchmark benchmark_main) + if(NOT CODSPEED_MODE) + FetchContent_Declare( + benchmark + URL https://github.com/google/benchmark/archive/refs/tags/v1.9.5.tar.gz + URL_HASH SHA256=9631341c82bac4a288bef951f8b26b41f69021794184ece969f8473977eaa340 + ) + else() + # Replace google/benchmark with CodSpeed's fork of it to report the benchmark results + # to https://codspeed.io. It provides the same benchmark::benchmark target, + # so the benchmark targets need no changes. + FetchContent_Declare( + benchmark + # Fork of v2.4.0 with fixes not yet available upstream. + GIT_REPOSITORY https://github.com/chfast/codspeed-cpp + GIT_TAG 0ecdd6e7d8f9e858e8e0b2fbe54ff7f427bec528 + GIT_SHALLOW FALSE + SOURCE_SUBDIR google_benchmark + ) + list(APPEND benchmark_libs codspeed instrument_hooks) + endif() FetchContent_MakeAvailable(benchmark) endblock() + set_target_properties( - benchmark benchmark_main PROPERTIES - COMPILE_OPTIONS $<$:-w> + ${benchmark_libs} PROPERTIES + COMPILE_OPTIONS "$<$:-w>;$<$:-w>" + C_CLANG_TIDY "" CXX_CLANG_TIDY "" SYSTEM TRUE # TODO(cmake-3.26): Use the SYSTEM option of FetchContent_Declare(). ) diff --git a/test/bench/bench.cpp b/test/bench/bench.cpp index 8c45f83693..f6ef1e40e2 100644 --- a/test/bench/bench.cpp +++ b/test/bench/bench.cpp @@ -116,7 +116,7 @@ void register_benchmarks(std::span benchmark_cases) { if (advanced_vm != nullptr) { - RegisterBenchmark("advanced/analyse/" + b.name, [&b](State& state) { + RegisterBenchmark(bench_name("advanced/analyse/" + b.name), [&b](State& state) { bench_analyse( state, default_revision, b.code); })->Unit(kMicrosecond); @@ -124,7 +124,7 @@ void register_benchmarks(std::span benchmark_cases) if (baseline_vm != nullptr) { - RegisterBenchmark("baseline/analyse/" + b.name, [&b](State& state) { + RegisterBenchmark(bench_name("baseline/analyse/" + b.name), [&b](State& state) { bench_analyse( state, default_revision, b.code); })->Unit(kMicrosecond); @@ -136,7 +136,7 @@ void register_benchmarks(std::span benchmark_cases) if (advanced_vm != nullptr) { - const auto name = "advanced/execute/" + case_name; + const auto name = bench_name("advanced/execute/" + case_name); RegisterBenchmark(name, [&vm = *advanced_vm, &b, &input](State& state) { bench_advanced_execute(state, vm, b.code, input.input, input.expected_output); })->Unit(kMicrosecond); @@ -144,7 +144,7 @@ void register_benchmarks(std::span benchmark_cases) if (baseline_vm != nullptr) { - const auto name = "baseline/execute/" + case_name; + const auto name = bench_name("baseline/execute/" + case_name); RegisterBenchmark(name, [&vm = *baseline_vm, &b, &input](State& state) { bench_baseline_execute(state, vm, b.code, input.input, input.expected_output); })->Unit(kMicrosecond); @@ -152,7 +152,7 @@ void register_benchmarks(std::span benchmark_cases) if (basel_cg_vm != nullptr) { - const auto name = "bnocgoto/execute/" + case_name; + const auto name = bench_name("bnocgoto/execute/" + case_name); RegisterBenchmark(name, [&vm = *basel_cg_vm, &b, &input](State& state) { bench_baseline_execute(state, vm, b.code, input.input, input.expected_output); })->Unit(kMicrosecond); @@ -160,7 +160,7 @@ void register_benchmarks(std::span benchmark_cases) for (auto& [vm_name, vm] : registered_vms) { - const auto name = std::string{vm_name} + "/total/" + case_name; + const auto name = bench_name(std::string{vm_name} + "/total/" + case_name); RegisterBenchmark(name, [&vm, &b, &input](State& state) { bench_evmc_execute(state, vm, b.code, input.input, input.expected_output); })->Unit(kMicrosecond); @@ -234,7 +234,10 @@ std::tuple> parseargs(int argc, char** argv) int main(int argc, char** argv) { +#ifndef CODSPEED_ENABLED + // Not available in the google/benchmark version CodSpeed's fork is based on, nor needed there. MaybeReenterWithoutASLR(argc, argv); +#endif using namespace evmone::test; try diff --git a/test/bench/helpers.hpp b/test/bench/helpers.hpp index 2f0f71a0a8..650fa541b4 100644 --- a/test/bench/helpers.hpp +++ b/test/bench/helpers.hpp @@ -11,11 +11,26 @@ #include #include #include +#include namespace evmone::test { extern std::map registered_vms; +/// Decorates a dynamically registered benchmark name with the location of the registration. +/// CodSpeed identifies benchmarks by the "source_file::name" URI, which the BENCHMARK() macro +/// adds automatically, but RegisterBenchmark() does not. Without CodSpeed this is a no-op. +/// The default argument is evaluated at the call site, naming the file registering the benchmark. +inline std::string bench_name(std::string name, + [[maybe_unused]] const std::source_location loc = std::source_location::current()) +{ +#ifdef CODSPEED_ENABLED + return codspeed::get_path_relative_to_workspace(loc.file_name()) + "::" + std::move(name); +#else + return name; +#endif +} + constexpr auto default_revision = EVMC_ISTANBUL; constexpr auto default_gas_limit = std::numeric_limits::max(); diff --git a/test/bench/synthetic_benchmarks.cpp b/test/bench/synthetic_benchmarks.cpp index 0aaa7d9243..f564bd363d 100644 --- a/test/bench/synthetic_benchmarks.cpp +++ b/test/bench/synthetic_benchmarks.cpp @@ -252,9 +252,9 @@ void register_synthetic_benchmarks() for (auto& [vm_name, vm] : registered_vms) { - RegisterBenchmark(std::string{vm_name} + "/total/synth/loop_v1", + RegisterBenchmark(bench_name(std::string{vm_name} + "/total/synth/loop_v1"), [&vm](State& state) { bench_evmc_execute(state, vm, generate_loop_v1({})); }); - RegisterBenchmark(std::string{vm_name} + "/total/synth/loop_v2", + RegisterBenchmark(bench_name(std::string{vm_name} + "/total/synth/loop_v2"), [&vm](State& state) { bench_evmc_execute(state, vm, generate_loop_v2({})); }); } @@ -262,7 +262,8 @@ void register_synthetic_benchmarks() { for (auto& [vm_name, vm] : registered_vms) { - RegisterBenchmark(std::string{vm_name} + "/total/synth/" + to_string(params), + RegisterBenchmark( + bench_name(std::string{vm_name} + "/total/synth/" + to_string(params)), [&vm, params]( State& state) { bench_evmc_execute(state, vm, generate_code(params)); }) ->Unit(kMicrosecond);