From 3b474e31fdd64c4188ead158e403556c5017f074 Mon Sep 17 00:00:00 2001 From: Max Tropets Date: Fri, 4 Sep 2026 13:56:35 +0100 Subject: [PATCH 1/2] Only build required targets for bencher Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 41b05883-5e0c-4bde-a945-0de0ceccde8e --- .github/workflows/bencher.yml | 46 +++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bencher.yml b/.github/workflows/bencher.yml index 7f07635c993..5e2e0378636 100644 --- a/.github/workflows/bencher.yml +++ b/.github/workflows/bencher.yml @@ -88,11 +88,32 @@ jobs: - name: Build and run virtual perf tests run: | + set -eo pipefail git config --global --add safe.directory /__w/CCF/CCF mkdir build cd build cmake -GNinja -DWORKER_THREADS=2 .. - ninja + # Build only the binaries the benchmark and perf suites actually run. + # A bare `ninja` also builds every unit-test binary, which this job + # never executes and which dominates the build. + # add_picobench names each test after its executable target + readarray -t bench_targets < <( + ctest -N -L benchmark | sed -n 's/^ *Test *#[0-9]*: //p' + ) + # perf tests name the app they need on their command line + readarray -t app_targets < <( + ctest -N -V -L perf -C perf | tr ' ' '\n' | tr -d '"' | awk ' + /^--package$/ { getline; print } + /^-c$/ { getline; sub(/^\.\//, ""); print } + /^--js-app-bundle$/ { print "js_generic" }' | sort -u + ) + # Guard each half separately: the benchmark names alone would satisfy + # a combined count check even if app discovery silently returned none. + if [ "${#bench_targets[@]}" -eq 0 ] || [ "${#app_targets[@]}" -eq 0 ]; then + echo "Derived ${#bench_targets[@]} benchmark and ${#app_targets[@]} app targets, expected both non-empty" >&2 + exit 1 + fi + ninja "${bench_targets[@]}" "${app_targets[@]}" # Microbenchmarks ./tests.sh -VV -L benchmark # End to end performance tests @@ -177,11 +198,32 @@ jobs: - name: Build and run SNP perf tests run: | + set -eo pipefail git config --global --add safe.directory /__w/CCF/CCF mkdir build cd build cmake -GNinja -DWORKER_THREADS=2 .. - ninja + # Build only the binaries the benchmark and perf suites actually run. + # A bare `ninja` also builds every unit-test binary, which this job + # never executes and which dominates the build. + # add_picobench names each test after its executable target + readarray -t bench_targets < <( + ctest -N -L benchmark | sed -n 's/^ *Test *#[0-9]*: //p' + ) + # perf tests name the app they need on their command line + readarray -t app_targets < <( + ctest -N -V -L perf -C perf | tr ' ' '\n' | tr -d '"' | awk ' + /^--package$/ { getline; print } + /^-c$/ { getline; sub(/^\.\//, ""); print } + /^--js-app-bundle$/ { print "js_generic" }' | sort -u + ) + # Guard each half separately: the benchmark names alone would satisfy + # a combined count check even if app discovery silently returned none. + if [ "${#bench_targets[@]}" -eq 0 ] || [ "${#app_targets[@]}" -eq 0 ]; then + echo "Derived ${#bench_targets[@]} benchmark and ${#app_targets[@]} app targets, expected both non-empty" >&2 + exit 1 + fi + ninja "${bench_targets[@]}" "${app_targets[@]}" # Microbenchmarks ./tests.sh -VV -L benchmark -E task_bench # End to end performance tests From 29e1b441dab68827ed07871a30acd70a486a3d78 Mon Sep 17 00:00:00 2001 From: Max Tropets Date: Mon, 7 Sep 2026 11:21:46 +0100 Subject: [PATCH 2/2] As presets --- .github/workflows/README.md | 16 +++++++++++ .github/workflows/bencher.yml | 50 +++------------------------------ .gitignore | 1 + CMakeLists.txt | 8 ++++++ CMakePresets.json | 28 ++++++++++++++++++ cmake/common.cmake | 13 ++++++++- cmake/gersemi_definitions.cmake | 2 +- 7 files changed, 70 insertions(+), 48 deletions(-) create mode 100644 CMakePresets.json diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 4eb03077223..bfab0167149 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -19,6 +19,22 @@ Triggered on every commit on `main`, twice daily on week days, and manually, but Tests are run on two different testbeds for comparison: gha-vmss-d16av6-ci (d16av6 VMs) and gha-c-aci-ci (C-ACI with 16 cores and 32Gb RAM). +Both jobs use the `bencher` configure and build presets in [`CMakePresets.json`](../../CMakePresets.json). With the build dependencies installed, run these commands from the repository root to use the same build settings locally: + +```bash +cmake --preset bencher +cmake --build --preset bencher +cd build +./tests.sh -VV -L benchmark +./tests.sh -VV -L perf -C perf +``` + +On SNP, add `-E task_bench` to the microbenchmark command to match the workflow. + +The configure preset selects Ninja, `RelWithDebInfo`, and two worker threads, using the existing `build/` directory. Additional configure options can still be supplied, for example `cmake --preset bencher -DWORKER_THREADS=4`. Local presets can be defined in the ignored `CMakeUserPresets.json` file. + +The build preset selects the `ccf_bencher` aggregate target rather than building every target. `add_picobench()` automatically registers its executable with this target. Each perf `add_e2e_test()` must declare its dependencies with `BUILD_TARGETS`, using CMake target names such as `logging`, not executable paths such as `samples/apps/logging/logging`. Declare these dependencies alongside each perf test so the build follows the registered tests without parsing CTest output. Normal builds without presets remain unchanged. + File: `bencher.yml` 3rd party dependencies: None diff --git a/.github/workflows/bencher.yml b/.github/workflows/bencher.yml index 5e2e0378636..b0e184bc30a 100644 --- a/.github/workflows/bencher.yml +++ b/.github/workflows/bencher.yml @@ -90,30 +90,9 @@ jobs: run: | set -eo pipefail git config --global --add safe.directory /__w/CCF/CCF - mkdir build + cmake --preset bencher + cmake --build --preset bencher cd build - cmake -GNinja -DWORKER_THREADS=2 .. - # Build only the binaries the benchmark and perf suites actually run. - # A bare `ninja` also builds every unit-test binary, which this job - # never executes and which dominates the build. - # add_picobench names each test after its executable target - readarray -t bench_targets < <( - ctest -N -L benchmark | sed -n 's/^ *Test *#[0-9]*: //p' - ) - # perf tests name the app they need on their command line - readarray -t app_targets < <( - ctest -N -V -L perf -C perf | tr ' ' '\n' | tr -d '"' | awk ' - /^--package$/ { getline; print } - /^-c$/ { getline; sub(/^\.\//, ""); print } - /^--js-app-bundle$/ { print "js_generic" }' | sort -u - ) - # Guard each half separately: the benchmark names alone would satisfy - # a combined count check even if app discovery silently returned none. - if [ "${#bench_targets[@]}" -eq 0 ] || [ "${#app_targets[@]}" -eq 0 ]; then - echo "Derived ${#bench_targets[@]} benchmark and ${#app_targets[@]} app targets, expected both non-empty" >&2 - exit 1 - fi - ninja "${bench_targets[@]}" "${app_targets[@]}" # Microbenchmarks ./tests.sh -VV -L benchmark # End to end performance tests @@ -200,30 +179,9 @@ jobs: run: | set -eo pipefail git config --global --add safe.directory /__w/CCF/CCF - mkdir build + cmake --preset bencher + cmake --build --preset bencher cd build - cmake -GNinja -DWORKER_THREADS=2 .. - # Build only the binaries the benchmark and perf suites actually run. - # A bare `ninja` also builds every unit-test binary, which this job - # never executes and which dominates the build. - # add_picobench names each test after its executable target - readarray -t bench_targets < <( - ctest -N -L benchmark | sed -n 's/^ *Test *#[0-9]*: //p' - ) - # perf tests name the app they need on their command line - readarray -t app_targets < <( - ctest -N -V -L perf -C perf | tr ' ' '\n' | tr -d '"' | awk ' - /^--package$/ { getline; print } - /^-c$/ { getline; sub(/^\.\//, ""); print } - /^--js-app-bundle$/ { print "js_generic" }' | sort -u - ) - # Guard each half separately: the benchmark names alone would satisfy - # a combined count check even if app discovery silently returned none. - if [ "${#bench_targets[@]}" -eq 0 ] || [ "${#app_targets[@]}" -eq 0 ]; then - echo "Derived ${#bench_targets[@]} benchmark and ${#app_targets[@]} app targets, expected both non-empty" >&2 - exit 1 - fi - ninja "${bench_targets[@]}" "${app_targets[@]}" # Microbenchmarks ./tests.sh -VV -L benchmark -E task_bench # End to end performance tests diff --git a/.gitignore b/.gitignore index c7f06416c16..482f3c5a73b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ build*/ Debug*/ Release*/ CMakeFiles/ +/CMakeUserPresets.json .vscode/ .vs/ libuv/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 44146af74d7..d623396f21a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -573,6 +573,8 @@ install(TARGETS verify_attestation DESTINATION bin) if(BUILD_TESTS) enable_testing() + add_custom_target(ccf_bencher) + # Unit tests if(BUILD_UNIT_TESTS) add_test( @@ -1202,6 +1204,7 @@ if(BUILD_TESTS) PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/commit_latency.py LABEL perf CONFIGURATIONS perf + BUILD_TARGETS logging ) add_e2e_test( @@ -1459,6 +1462,7 @@ if(BUILD_TESTS) PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/logging_cert_locust.py LABEL perf CONFIGURATIONS perf + BUILD_TARGETS logging ADDITIONAL_ARGS --package "samples/apps/logging/logging" @@ -1485,6 +1489,7 @@ if(BUILD_TESTS) PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/basicperf_locust.py LABEL perf CONFIGURATIONS perf + BUILD_TARGETS js_generic ADDITIONAL_ARGS --js-app-bundle ${CMAKE_SOURCE_DIR}/samples/apps/basic/js @@ -1514,6 +1519,7 @@ if(BUILD_TESTS) PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/basicperf_locust.py LABEL perf CONFIGURATIONS perf + BUILD_TARGETS basic ADDITIONAL_ARGS --package "samples/apps/basic/basic" @@ -1545,6 +1551,7 @@ if(BUILD_TESTS) PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/logging_jwt_locust.py LABEL perf CONFIGURATIONS perf + BUILD_TARGETS logging ADDITIONAL_ARGS --package "samples/apps/logging/logging" @@ -1570,6 +1577,7 @@ if(BUILD_TESTS) PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/historical_query_locust.py LABEL perf CONFIGURATIONS perf + BUILD_TARGETS logging ADDITIONAL_ARGS --package "samples/apps/logging/logging" diff --git a/CMakePresets.json b/CMakePresets.json new file mode 100644 index 00000000000..b3a7352d366 --- /dev/null +++ b/CMakePresets.json @@ -0,0 +1,28 @@ +{ + "version": 3, + "cmakeMinimumRequired": { + "major": 3, + "minor": 21, + "patch": 0 + }, + "configurePresets": [ + { + "name": "bencher", + "displayName": "Bencher", + "description": "Configure the microbenchmark and end-to-end performance test build.", + "generator": "Ninja", + "binaryDir": "${sourceDir}/build", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "RelWithDebInfo", + "WORKER_THREADS": "2" + } + } + ], + "buildPresets": [ + { + "name": "bencher", + "configurePreset": "bencher", + "targets": ["ccf_bencher"] + } + ] +} diff --git a/cmake/common.cmake b/cmake/common.cmake index 138cddc4687..fb4279487ab 100644 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -141,7 +141,7 @@ function(add_e2e_test) PARSED_ARGS "DETECT_DEADLOCKS" "NAME;PYTHON_SCRIPT;LABEL;CURL_CLIENT;BUCKET;TSAN_SUPPRESSIONS" - "CONSTITUTION;ADDITIONAL_ARGS;CONFIGURATIONS" + "CONSTITUTION;ADDITIONAL_ARGS;CONFIGURATIONS;BUILD_TARGETS" ) if(NOT PARSED_ARGS_CONSTITUTION) @@ -149,6 +149,16 @@ function(add_e2e_test) endif() if(BUILD_END_TO_END_TESTS) + if("${PARSED_ARGS_LABEL}" STREQUAL "perf") + if(NOT PARSED_ARGS_BUILD_TARGETS) + message( + FATAL_ERROR + "Perf test ${PARSED_ARGS_NAME} must specify BUILD_TARGETS" + ) + endif() + add_dependencies(ccf_bencher ${PARSED_ARGS_BUILD_TARGETS}) + endif() + set(PYTHON_WRAPPER ${PYTHON}) # For fast e2e runs, tick node faster than default value (except for @@ -262,6 +272,7 @@ function(add_picobench name) ) add_executable(${name} ${PARSED_ARGS_SRCS}) + add_dependencies(ccf_bencher ${name}) target_include_directories(${name} PRIVATE src ${PARSED_ARGS_INCLUDE_DIRS}) diff --git a/cmake/gersemi_definitions.cmake b/cmake/gersemi_definitions.cmake index 471cf7395e1..848a667e67b 100644 --- a/cmake/gersemi_definitions.cmake +++ b/cmake/gersemi_definitions.cmake @@ -25,7 +25,7 @@ function(add_e2e_test) PARSED_ARGS "DETECT_DEADLOCKS" "NAME;PYTHON_SCRIPT;LABEL;CURL_CLIENT;BUCKET;TSAN_SUPPRESSIONS" - "CONSTITUTION;ADDITIONAL_ARGS;CONFIGURATIONS" + "CONSTITUTION;ADDITIONAL_ARGS;CONFIGURATIONS;BUILD_TARGETS" ) endfunction()