diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 4eb03077223..bfab0167149 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -19,6 +19,22 @@ Triggered on every commit on `main`, twice daily on week days, and manually, but Tests are run on two different testbeds for comparison: gha-vmss-d16av6-ci (d16av6 VMs) and gha-c-aci-ci (C-ACI with 16 cores and 32Gb RAM). +Both jobs use the `bencher` configure and build presets in [`CMakePresets.json`](../../CMakePresets.json). With the build dependencies installed, run these commands from the repository root to use the same build settings locally: + +```bash +cmake --preset bencher +cmake --build --preset bencher +cd build +./tests.sh -VV -L benchmark +./tests.sh -VV -L perf -C perf +``` + +On SNP, add `-E task_bench` to the microbenchmark command to match the workflow. + +The configure preset selects Ninja, `RelWithDebInfo`, and two worker threads, using the existing `build/` directory. Additional configure options can still be supplied, for example `cmake --preset bencher -DWORKER_THREADS=4`. Local presets can be defined in the ignored `CMakeUserPresets.json` file. + +The build preset selects the `ccf_bencher` aggregate target rather than building every target. `add_picobench()` automatically registers its executable with this target. Each perf `add_e2e_test()` must declare its dependencies with `BUILD_TARGETS`, using CMake target names such as `logging`, not executable paths such as `samples/apps/logging/logging`. Declare these dependencies alongside each perf test so the build follows the registered tests without parsing CTest output. Normal builds without presets remain unchanged. + File: `bencher.yml` 3rd party dependencies: None diff --git a/.github/workflows/bencher.yml b/.github/workflows/bencher.yml index ddaf14f6756..9ce64904ff1 100644 --- a/.github/workflows/bencher.yml +++ b/.github/workflows/bencher.yml @@ -88,11 +88,11 @@ jobs: - name: Build and run virtual perf tests run: | + set -eo pipefail git config --global --add safe.directory /__w/CCF/CCF - mkdir build + cmake --preset bencher + cmake --build --preset bencher cd build - cmake -GNinja -DWORKER_THREADS=2 .. - ninja # Microbenchmarks ./tests.sh -VV -L benchmark # End to end performance tests @@ -178,11 +178,11 @@ jobs: - name: Build and run SNP perf tests run: | + set -eo pipefail git config --global --add safe.directory /__w/CCF/CCF - mkdir build + cmake --preset bencher + cmake --build --preset bencher cd build - cmake -GNinja -DWORKER_THREADS=2 .. - ninja # Microbenchmarks ./tests.sh -VV -L benchmark -E task_bench # End to end performance tests diff --git a/.gitignore b/.gitignore index c7f06416c16..482f3c5a73b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ build*/ Debug*/ Release*/ CMakeFiles/ +/CMakeUserPresets.json .vscode/ .vs/ libuv/ diff --git a/CMakeLists.txt b/CMakeLists.txt index dee27f1f816..dc0fc58d67d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -573,6 +573,8 @@ install(TARGETS verify_attestation DESTINATION bin) if(BUILD_TESTS) enable_testing() + add_custom_target(ccf_bencher) + # Unit tests if(BUILD_UNIT_TESTS) add_test( @@ -1208,6 +1210,7 @@ if(BUILD_TESTS) PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/commit_latency.py LABEL perf CONFIGURATIONS perf + BUILD_TARGETS logging ) add_e2e_test( @@ -1465,6 +1468,7 @@ if(BUILD_TESTS) PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/logging_cert_locust.py LABEL perf CONFIGURATIONS perf + BUILD_TARGETS logging ADDITIONAL_ARGS --package "samples/apps/logging/logging" @@ -1491,6 +1495,7 @@ if(BUILD_TESTS) PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/basicperf_locust.py LABEL perf CONFIGURATIONS perf + BUILD_TARGETS js_generic ADDITIONAL_ARGS --js-app-bundle ${CMAKE_SOURCE_DIR}/samples/apps/basic/js @@ -1520,6 +1525,7 @@ if(BUILD_TESTS) PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/basicperf_locust.py LABEL perf CONFIGURATIONS perf + BUILD_TARGETS basic ADDITIONAL_ARGS --package "samples/apps/basic/basic" @@ -1551,6 +1557,7 @@ if(BUILD_TESTS) PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/logging_jwt_locust.py LABEL perf CONFIGURATIONS perf + BUILD_TARGETS logging ADDITIONAL_ARGS --package "samples/apps/logging/logging" @@ -1576,6 +1583,7 @@ if(BUILD_TESTS) PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/historical_query_locust.py LABEL perf CONFIGURATIONS perf + BUILD_TARGETS logging ADDITIONAL_ARGS --package "samples/apps/logging/logging" diff --git a/CMakePresets.json b/CMakePresets.json new file mode 100644 index 00000000000..b3a7352d366 --- /dev/null +++ b/CMakePresets.json @@ -0,0 +1,28 @@ +{ + "version": 3, + "cmakeMinimumRequired": { + "major": 3, + "minor": 21, + "patch": 0 + }, + "configurePresets": [ + { + "name": "bencher", + "displayName": "Bencher", + "description": "Configure the microbenchmark and end-to-end performance test build.", + "generator": "Ninja", + "binaryDir": "${sourceDir}/build", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "RelWithDebInfo", + "WORKER_THREADS": "2" + } + } + ], + "buildPresets": [ + { + "name": "bencher", + "configurePreset": "bencher", + "targets": ["ccf_bencher"] + } + ] +} diff --git a/cmake/common.cmake b/cmake/common.cmake index 138cddc4687..fb4279487ab 100644 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -141,7 +141,7 @@ function(add_e2e_test) PARSED_ARGS "DETECT_DEADLOCKS" "NAME;PYTHON_SCRIPT;LABEL;CURL_CLIENT;BUCKET;TSAN_SUPPRESSIONS" - "CONSTITUTION;ADDITIONAL_ARGS;CONFIGURATIONS" + "CONSTITUTION;ADDITIONAL_ARGS;CONFIGURATIONS;BUILD_TARGETS" ) if(NOT PARSED_ARGS_CONSTITUTION) @@ -149,6 +149,16 @@ function(add_e2e_test) endif() if(BUILD_END_TO_END_TESTS) + if("${PARSED_ARGS_LABEL}" STREQUAL "perf") + if(NOT PARSED_ARGS_BUILD_TARGETS) + message( + FATAL_ERROR + "Perf test ${PARSED_ARGS_NAME} must specify BUILD_TARGETS" + ) + endif() + add_dependencies(ccf_bencher ${PARSED_ARGS_BUILD_TARGETS}) + endif() + set(PYTHON_WRAPPER ${PYTHON}) # For fast e2e runs, tick node faster than default value (except for @@ -262,6 +272,7 @@ function(add_picobench name) ) add_executable(${name} ${PARSED_ARGS_SRCS}) + add_dependencies(ccf_bencher ${name}) target_include_directories(${name} PRIVATE src ${PARSED_ARGS_INCLUDE_DIRS}) diff --git a/cmake/gersemi_definitions.cmake b/cmake/gersemi_definitions.cmake index 471cf7395e1..848a667e67b 100644 --- a/cmake/gersemi_definitions.cmake +++ b/cmake/gersemi_definitions.cmake @@ -25,7 +25,7 @@ function(add_e2e_test) PARSED_ARGS "DETECT_DEADLOCKS" "NAME;PYTHON_SCRIPT;LABEL;CURL_CLIENT;BUCKET;TSAN_SUPPRESSIONS" - "CONSTITUTION;ADDITIONAL_ARGS;CONFIGURATIONS" + "CONSTITUTION;ADDITIONAL_ARGS;CONFIGURATIONS;BUILD_TARGETS" ) endfunction()