From 75e2a9b9a06afb3421a28d88c92abc96b757d9b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Noah=20Tr=E1=BA=A7n?= Date: Tue, 20 Oct 2026 21:33:33 +0700 Subject: [PATCH 1/2] feat(benchmark): harden harness and add small-host profile - add Graphify project integration for OpenCode and Codex - stop sampler tasks and managed containers on every exit path - reject unsafe manifest paths and validate result categories - fix deadline error accounting, PID CPU fallback, RAPL and RSS peaks - preallocate histograms and remove unused hot-path dependencies - add bounded 2 vCPU / 8 GB OpenStack wrapper, docs, matrix, CI smoke - refresh workspace and framework dependencies, toolchains, and actions --- .codex/agents/architect.toml | 2 +- .codex/config.toml | 6 + .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- .github/workflows/benchmark.yml | 23 +- .github/workflows/benchmark_small_smoke.yml | 43 + .github/workflows/benchmark_smoke.yml | 4 +- .github/workflows/cpp-lint.yml | 2 +- .github/workflows/csharp-lint.yml | 6 +- .github/workflows/go-lint.yml | 6 +- .github/workflows/java-lint.yml | 4 +- .github/workflows/javascript-lint.yml | 8 +- .github/workflows/lint.yml | 30 +- .github/workflows/php-lint.yml | 6 +- .github/workflows/python-lint.yml | 6 +- .github/workflows/ruby-lint.yml | 6 +- .github/workflows/rust-lint.yml | 7 +- .github/workflows/zig-lint.yml | 4 +- .gitignore | 12 + .graphifyignore | 7 + .opencode/opencode.json | 11 +- .opencode/plugins/graphify.js | 29 + AGENTS.md | 2 +- CODEBASE.md | 2 + CODING_CONVENTION.md | 2 +- Cargo.lock | 221 ++---- Cargo.toml | 19 +- README.md | 4 + TODO.md | 2 + crates/collector/src/aggregator.rs | 2 +- crates/collector/src/ebpf.rs | 4 +- crates/collector/src/lib.rs | 5 +- crates/collector/src/procfs.rs | 69 +- crates/collector/src/rapl.rs | 31 +- crates/collector/src/sampler.rs | 135 ++-- crates/collector/src/sampler/sources.rs | 93 +++ crates/collector/tests/collector_test.rs | 21 + crates/collector/tests/sampler_test.rs | 2 +- crates/common/src/lib.rs | 5 +- crates/common/src/manifest.rs | 62 +- crates/common/src/result.rs | 2 +- crates/common/src/validation.rs | 156 ++++ crates/common/src/validation_build.rs | 95 +++ crates/common/tests/manifest_test.rs | 33 + crates/load-gen/Cargo.toml | 1 - crates/load-gen/src/autobahn/builder.rs | 2 +- crates/load-gen/src/autobahn/frame.rs | 6 +- crates/load-gen/src/autobahn/fuzz.rs | 2 +- crates/load-gen/src/autobahn/runner.rs | 49 +- crates/load-gen/src/client.rs | 22 +- crates/load-gen/src/engine.rs | 16 +- crates/load-gen/src/error.rs | 3 + crates/load-gen/src/histogram.rs | 6 - crates/load-gen/src/plan.rs | 58 +- crates/load-gen/src/response.rs | 3 +- crates/load-gen/tests/load_gen_test.rs | 4 +- crates/orchestrator/src/build_options.rs | 18 +- crates/orchestrator/src/container.rs | 107 ++- crates/orchestrator/src/container_args.rs | 121 +++ crates/orchestrator/src/container_config.rs | 20 +- crates/orchestrator/src/container_guard.rs | 33 + crates/orchestrator/src/container_paths.rs | 69 ++ crates/orchestrator/src/database.rs | 5 +- crates/orchestrator/src/discovery.rs | 37 +- crates/orchestrator/src/host_profile.rs | 94 +++ crates/orchestrator/src/lib.rs | 6 + crates/orchestrator/src/lifecycle.rs | 20 +- crates/orchestrator/src/main.rs | 20 +- crates/orchestrator/src/poller.rs | 41 +- crates/orchestrator/src/provenance.rs | 8 +- crates/orchestrator/src/runner.rs | 128 ++- crates/orchestrator/src/session_results.rs | 27 + crates/orchestrator/src/suite_timing.rs | 44 ++ crates/orchestrator/src/workload.rs | 40 +- crates/orchestrator/src/ws_bench.rs | 109 ++- crates/orchestrator/tests/discovery_test.rs | 22 +- crates/reporter/src/archive.rs | 21 +- crates/reporter/src/artifacts.rs | 38 + crates/reporter/src/chart.rs | 48 +- crates/reporter/src/history.rs | 31 +- crates/reporter/src/lib.rs | 4 + crates/reporter/src/markdown.rs | 145 +--- crates/reporter/src/markdown/tables.rs | 161 ++++ crates/reporter/src/safety.rs | 127 +++ crates/reporter/src/storage.rs | 2 +- crates/reporter/tests/artifacts_test.rs | 62 ++ docs/architecture.md | 4 +- docs/graphify.md | 91 +++ docs/how_to_use.md | 12 +- docs/roadmap/acceptance_criteria.md | 2 +- docs/small_host.md | 186 +++++ docs/small_host_frameworks.md | 84 ++ flake.lock | 12 +- flake.nix | 14 +- .../web-server/cpp/agensio/Dockerfile.glibc | 3 +- .../web-server/cpp/agensio/Dockerfile.musl | 7 +- .../web-server/cpp/agensio/framework.toml | 2 +- .../web-server/cpp/drogon/Dockerfile.glibc | 7 +- .../web-server/cpp/drogon/Dockerfile.musl | 7 +- .../web-server/cpp/drogon/framework.toml | 2 +- .../web-server/cpp/hical/Dockerfile.glibc | 7 +- .../web-server/cpp/hical/Dockerfile.musl | 7 +- .../web-server/cpp/hical/framework.toml | 2 +- .../cpp/uwebsockets/Dockerfile.glibc | 7 +- .../cpp/uwebsockets/Dockerfile.musl | 7 +- .../web-server/cpp/uwebsockets/framework.toml | 2 +- .../csharp/aspnet-minimal/Dockerfile.glibc | 5 +- .../csharp/aspnet-minimal/Dockerfile.musl | 5 +- .../aspnet-minimal/aspnet-minimal.csproj | 2 +- .../csharp/aspnet-minimal/framework.toml | 4 +- .../csharp/cadente/Dockerfile.glibc | 5 +- .../web-server/csharp/cadente/Dockerfile.musl | 5 +- .../web-server/csharp/cadente/Program.cs | 2 +- .../web-server/csharp/cadente/cadente.csproj | 6 +- .../web-server/csharp/cadente/framework.toml | 4 +- .../csharp/genhttp/Dockerfile.glibc | 5 +- .../web-server/csharp/genhttp/Dockerfile.musl | 5 +- .../web-server/csharp/genhttp/Program.cs | 2 +- .../web-server/csharp/genhttp/framework.toml | 4 +- .../web-server/csharp/genhttp/genhttp.csproj | 6 +- .../web-server/go/fasthttp/Dockerfile.glibc | 7 +- .../web-server/go/fasthttp/Dockerfile.musl | 7 +- .../web-server/go/fasthttp/framework.toml | 4 +- frameworks/web-server/go/fasthttp/go.mod | 8 +- frameworks/web-server/go/fasthttp/go.sum | 16 +- .../web-server/go/fib-tuned/Dockerfile.glibc | 7 +- .../web-server/go/fib-tuned/Dockerfile.musl | 7 +- .../web-server/go/fib-tuned/framework.toml | 4 +- frameworks/web-server/go/fib-tuned/go.mod | 26 +- frameworks/web-server/go/fib-tuned/go.sum | 63 +- frameworks/web-server/go/fib-tuned/main.go | 18 +- .../web-server/go/fiber/Dockerfile.glibc | 7 +- .../web-server/go/fiber/Dockerfile.musl | 7 +- frameworks/web-server/go/fiber/framework.toml | 4 +- frameworks/web-server/go/fiber/go.mod | 26 +- frameworks/web-server/go/fiber/go.sum | 63 +- frameworks/web-server/go/fiber/main.go | 14 +- .../web-server/java/rapidoid/Dockerfile.glibc | 5 +- .../web-server/java/rapidoid/Dockerfile.musl | 5 +- frameworks/web-server/java/rapidoid/pom.xml | 2 +- .../web-server/java/undertow/Dockerfile.glibc | 5 +- .../web-server/java/undertow/Dockerfile.musl | 5 +- .../web-server/java/undertow/framework.toml | 2 +- frameworks/web-server/java/undertow/pom.xml | 4 +- .../web-server/java/vertx/Dockerfile.glibc | 5 +- .../web-server/java/vertx/Dockerfile.musl | 5 +- .../web-server/java/vertx/framework.toml | 2 +- frameworks/web-server/java/vertx/pom.xml | 4 +- .../javascript/bun/Dockerfile.glibc | 3 +- .../web-server/javascript/bun/Dockerfile.musl | 3 +- .../web-server/javascript/bun/framework.toml | 4 +- .../javascript/elysia/Dockerfile.glibc | 5 +- .../javascript/elysia/Dockerfile.musl | 5 +- .../javascript/elysia/framework.toml | 4 +- .../web-server/javascript/elysia/package.json | 2 +- .../javascript/fulmine/Dockerfile.glibc | 8 +- .../javascript/fulmine/Dockerfile.musl | 8 +- .../javascript/fulmine/framework.toml | 4 +- .../javascript/fulmine/package-lock.json | 21 +- .../javascript/fulmine/package.json | 2 +- .../php/frankenphp/Dockerfile.glibc | 7 +- .../web-server/php/frankenphp/Dockerfile.musl | 7 +- .../web-server/php/frankenphp/framework.toml | 4 +- .../web-server/php/swoole/Dockerfile.glibc | 11 +- .../web-server/php/swoole/Dockerfile.musl | 10 +- .../web-server/php/swoole/framework.toml | 4 +- .../web-server/php/workerman/Dockerfile.glibc | 14 +- .../web-server/php/workerman/Dockerfile.musl | 13 +- .../web-server/php/workerman/composer.json | 2 +- .../web-server/php/workerman/framework.toml | 4 +- .../python/blacksheep/Dockerfile.glibc | 3 +- .../python/blacksheep/Dockerfile.musl | 8 +- .../web-server/python/blacksheep/app.py | 6 +- .../python/blacksheep/framework.toml | 4 +- .../python/blacksheep/requirements.txt | 8 +- .../python/granian/Dockerfile.glibc | 3 +- .../web-server/python/granian/Dockerfile.musl | 8 +- .../web-server/python/granian/framework.toml | 4 +- .../python/granian/requirements.txt | 4 +- .../web-server/python/robyn/Dockerfile.glibc | 3 +- .../web-server/python/robyn/Dockerfile.musl | 8 +- .../web-server/python/robyn/framework.toml | 4 +- .../web-server/python/robyn/requirements.txt | 2 +- .../web-server/ruby/agoo/Dockerfile.glibc | 10 +- .../web-server/ruby/agoo/Dockerfile.musl | 9 +- frameworks/web-server/ruby/agoo/Gemfile | 2 +- .../web-server/ruby/agoo/framework.toml | 4 +- frameworks/web-server/ruby/agoo/server.rb | 1 + .../web-server/ruby/falcon/Dockerfile.glibc | 10 +- .../web-server/ruby/falcon/Dockerfile.musl | 9 +- frameworks/web-server/ruby/falcon/Gemfile | 2 +- frameworks/web-server/ruby/falcon/config.ru | 4 +- .../web-server/ruby/falcon/framework.toml | 4 +- .../web-server/ruby/iodine/Dockerfile.glibc | 10 +- .../web-server/ruby/iodine/Dockerfile.musl | 9 +- frameworks/web-server/ruby/iodine/Gemfile | 2 +- .../web-server/ruby/iodine/framework.toml | 4 +- frameworks/web-server/ruby/iodine/server.rb | 2 +- .../web-server/rust/actix-web/Cargo.lock | 72 +- .../web-server/rust/actix-web/Cargo.toml | 4 +- .../rust/actix-web/Dockerfile.glibc | 7 +- .../web-server/rust/actix-web/Dockerfile.musl | 7 +- .../web-server/rust/actix-web/framework.toml | 4 +- frameworks/web-server/rust/cpeak/Cargo.lock | 69 +- frameworks/web-server/rust/cpeak/Cargo.toml | 2 +- .../web-server/rust/cpeak/Dockerfile.glibc | 7 +- .../web-server/rust/cpeak/Dockerfile.musl | 7 +- .../web-server/rust/cpeak/framework.toml | 4 +- frameworks/web-server/rust/ntex/Cargo.lock | 736 ++++++++++++------ frameworks/web-server/rust/ntex/Cargo.toml | 2 +- .../web-server/rust/ntex/Dockerfile.glibc | 7 +- .../web-server/rust/ntex/Dockerfile.musl | 7 +- .../web-server/rust/ntex/framework.toml | 4 +- frameworks/web-server/rust/ntex/src/main.rs | 7 +- .../web-server/zig/dusty/Dockerfile.glibc | 13 +- .../web-server/zig/dusty/Dockerfile.musl | 13 +- frameworks/web-server/zig/dusty/build.zig.zon | 2 +- .../web-server/zig/dusty/framework.toml | 2 +- frameworks/web-server/zig/dusty/src/main.zig | 12 +- .../web-server/zig/swerver/Dockerfile.glibc | 11 +- .../web-server/zig/swerver/Dockerfile.musl | 11 +- .../zig/uwebzockets/Dockerfile.glibc | 11 +- .../zig/uwebzockets/Dockerfile.musl | 11 +- scripts/check_conventions.ps1 | 4 +- scripts/check_conventions.sh | 6 +- scripts/run_autobahn_suite.sh | 4 + scripts/run_benchmarks_all.sh | 2 +- scripts/run_small_host.sh | 80 ++ scripts/validate_environment.sh | 11 + templates/docker/Dockerfile.glibc | 5 +- templates/docker/Dockerfile.musl | 5 +- 230 files changed, 3810 insertions(+), 1459 deletions(-) create mode 100644 .github/workflows/benchmark_small_smoke.yml create mode 100644 .graphifyignore create mode 100644 .opencode/plugins/graphify.js create mode 100644 crates/collector/src/sampler/sources.rs create mode 100644 crates/common/src/validation.rs create mode 100644 crates/common/src/validation_build.rs create mode 100644 crates/orchestrator/src/container_args.rs create mode 100644 crates/orchestrator/src/container_guard.rs create mode 100644 crates/orchestrator/src/container_paths.rs create mode 100644 crates/orchestrator/src/host_profile.rs create mode 100644 crates/orchestrator/src/session_results.rs create mode 100644 crates/orchestrator/src/suite_timing.rs create mode 100644 crates/reporter/src/artifacts.rs create mode 100644 crates/reporter/src/markdown/tables.rs create mode 100644 crates/reporter/src/safety.rs create mode 100644 crates/reporter/tests/artifacts_test.rs create mode 100644 docs/graphify.md create mode 100644 docs/small_host.md create mode 100644 docs/small_host_frameworks.md create mode 100755 scripts/run_small_host.sh diff --git a/.codex/agents/architect.toml b/.codex/agents/architect.toml index 6ba4219..cf3d895 100644 --- a/.codex/agents/architect.toml +++ b/.codex/agents/architect.toml @@ -4,5 +4,5 @@ sandbox_mode = "read-only" developer_instructions = """ Act as the project architect for framework-benchmark. Read AGENTS.md, CODING_CONVENTION.md, CODEBASE.md, and relevant specialist instructions before planning. Apply relevant skills from .agents/skills/ by reading their SKILL.md files. -Return explicit assumptions, scope, affected ownership boundaries, dependencies, a small ordered plan conforming to the 200 LOC per file hard ceiling, and verifiable acceptance criteria. Identify which existing domain specialist should guide or perform each slice. Keep the plan aligned with Rust 1.85+, zero-allocation hot paths, Data-Oriented Design, Functional Programming, Linux style, and zero emojis. Do not implement or edit files. +Return explicit assumptions, scope, affected ownership boundaries, dependencies, a small ordered plan conforming to the 200 LOC per file hard ceiling, and verifiable acceptance criteria. Identify which existing domain specialist should guide or perform each slice. Keep the plan aligned with Rust 1.90+, zero-allocation hot paths, Data-Oriented Design, Functional Programming, Linux style, and zero emojis. Do not implement or edit files. """ diff --git a/.codex/config.toml b/.codex/config.toml index 1e4efdb..28e52de 100644 --- a/.codex/config.toml +++ b/.codex/config.toml @@ -1,2 +1,8 @@ [agents] max_concurrent_threads_per_session = 4 + +[mcp_servers.graphify] +command = "graphify-mcp" +args = ["graphify-out/graph.json"] +cwd = "." +startup_timeout_sec = 20 diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 51ced0c..bf9c3b8 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -47,7 +47,7 @@ body: attributes: label: Environment & Toolchain description: Information about your OS, Kernel version, Rust version, Docker/Nix setup, and libc (glibc/musl). - placeholder: E.g. Ubuntu 24.04 (Linux 6.8), Rust 1.85, Docker 27.0, Nix 2.24 + placeholder: E.g. Ubuntu 24.04 (Linux 6.8), Rust 1.90, Docker 27.0, Nix 2.30 validations: required: true - type: textarea diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c5070d0..bb0e604 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -11,6 +11,9 @@ on: # Run nightly at 00:00 UTC - cron: '0 0 * * *' +permissions: + contents: read + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: false @@ -19,6 +22,7 @@ jobs: benchmark: name: Run Benchmark Suite runs-on: [self-hosted, linux, x64, benchmark-dedicated] + timeout-minutes: 360 env: BENCHMARK_FRAMEWORK_FILTER: ${{ inputs.framework_filter || 'all' }} BENCHMARK_DEDICATED_HOST: '1' @@ -27,13 +31,17 @@ jobs: BENCHMARK_CGROUP_PARENT: framework-benchmark steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false - name: Install stable Rust toolchain - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable + with: + toolchain: "1.90.0" - name: Cache cargo dependencies - uses: actions/cache@v4 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6 with: path: | ~/.cargo/registry/index/ @@ -48,13 +56,18 @@ jobs: run: sh scripts/verify_host.sh --require-dedicated - name: Build benchmark orchestrator - run: cargo build --release --bin orchestrator + run: cargo build --release --locked --bin orchestrator - name: Run orchestrator run: taskset --cpu-list "$BENCHMARK_LOAD_CPUS" ./target/release/orchestrator frameworks results + - name: Clean up benchmark resources + if: always() + continue-on-error: true + run: sh scripts/cleanup_environment.sh --apply + - name: Upload benchmark artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 with: name: benchmark-results path: | diff --git a/.github/workflows/benchmark_small_smoke.yml b/.github/workflows/benchmark_small_smoke.yml new file mode 100644 index 0000000..a4d9e0e --- /dev/null +++ b/.github/workflows/benchmark_small_smoke.yml @@ -0,0 +1,43 @@ +name: Benchmark Small-Host Smoke + +on: + workflow_dispatch: + schedule: + - cron: "0 3 * * 0" + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +jobs: + small-host: + name: Cpeak bounded smoke + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable + with: + toolchain: "1.90.0" + components: "clippy, rustfmt" + + - name: Cache cargo dependencies + uses: Swatinem/rust-cache@e9d7b9f1e587218970671bac74e5511ed9506e6a # v2 + + - name: Validate the bounded profile + run: scripts/run_small_host.sh --check-only + + - name: Build and run one framework + run: scripts/run_small_host.sh cpeak + + - name: Show generated artifacts + if: always() + run: find results -maxdepth 3 -type f 2>/dev/null | sort | head -50 diff --git a/.github/workflows/benchmark_smoke.yml b/.github/workflows/benchmark_smoke.yml index 5c4496f..de258fa 100644 --- a/.github/workflows/benchmark_smoke.yml +++ b/.github/workflows/benchmark_smoke.yml @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 20 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - name: Build and exercise both variants @@ -38,6 +38,7 @@ jobs: for libc in glibc musl; do image="benchmark-smoke:cpeak-${libc}" name="benchmark-smoke-${libc}" + trap 'docker stop --time 5 "$name" >/dev/null 2>&1 || true' EXIT docker build --file "Dockerfile.${libc}" --tag "$image" . docker run --detach --rm --network host --name "$name" "$image" for _ in $(seq 1 20); do @@ -46,4 +47,5 @@ jobs: done test "$(curl --fail --silent http://127.0.0.1:8080/plaintext)" = 'Hello, World!' docker stop --time 5 "$name" + trap - EXIT done diff --git a/.github/workflows/cpp-lint.yml b/.github/workflows/cpp-lint.yml index 1b6d5b8..cc44403 100644 --- a/.github/workflows/cpp-lint.yml +++ b/.github/workflows/cpp-lint.yml @@ -19,7 +19,7 @@ jobs: environment: name: lint-cpp steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - name: Install clang-format diff --git a/.github/workflows/csharp-lint.yml b/.github/workflows/csharp-lint.yml index 1fc4fe3..5e8a7f6 100644 --- a/.github/workflows/csharp-lint.yml +++ b/.github/workflows/csharp-lint.yml @@ -19,12 +19,12 @@ jobs: environment: name: lint-csharp steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - - uses: actions/setup-dotnet@v6 + - uses: actions/setup-dotnet@a98b56852c35b8e3190ac28c8c2271da59106c68 # v6 with: - dotnet-version: 9.0.x + dotnet-version: 10.0.x - name: Check formatting shell: bash run: | diff --git a/.github/workflows/go-lint.yml b/.github/workflows/go-lint.yml index 5f4c982..5fdab1b 100644 --- a/.github/workflows/go-lint.yml +++ b/.github/workflows/go-lint.yml @@ -19,12 +19,12 @@ jobs: environment: name: lint-go steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - - uses: actions/setup-go@v7 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7 with: - go-version: 1.23.0 + go-version: 1.26.10 check-latest: false cache-dependency-path: frameworks/web-server/go/*/go.mod - name: Check formatting diff --git a/.github/workflows/java-lint.yml b/.github/workflows/java-lint.yml index 70a4ef4..34ecb0b 100644 --- a/.github/workflows/java-lint.yml +++ b/.github/workflows/java-lint.yml @@ -19,10 +19,10 @@ jobs: environment: name: lint-java steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - - uses: actions/setup-java@v6 + - uses: actions/setup-java@b843ebf9b913146b38da2eaa7aeddf76074ae1ad # v6 with: distribution: temurin java-version: 21 diff --git a/.github/workflows/javascript-lint.yml b/.github/workflows/javascript-lint.yml index 97b7efe..ffdc5d5 100644 --- a/.github/workflows/javascript-lint.yml +++ b/.github/workflows/javascript-lint.yml @@ -19,11 +19,11 @@ jobs: environment: name: lint-javascript steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - - uses: oven-sh/setup-bun@v2 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.3.3 + bun-version: 1.4.3 - name: Check formatting - run: bunx --bun prettier@3.5.3 --check 'frameworks/web-server/javascript/**/*.{js,ts}' + run: bunx --bun prettier@3.9.10 --check 'frameworks/web-server/javascript/**/*.{js,ts}' diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 9897a50..685f1cd 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -5,44 +5,56 @@ on: branches: - main paths: + - "**/*.md" + - "**/*.toml" + - "**/*.yml" + - "**/*.yaml" - Cargo.toml - Cargo.lock - crates/** - - scripts/check_conventions.sh + - frameworks/** + - scripts/** + - templates/** - .github/workflows/lint.yml pull_request: branches: - main paths: + - "**/*.md" + - "**/*.toml" + - "**/*.yml" + - "**/*.yaml" - Cargo.toml - Cargo.lock - crates/** - - scripts/check_conventions.sh + - frameworks/** + - scripts/** + - templates/** - .github/workflows/lint.yml workflow_dispatch: +permissions: + contents: read + jobs: lint: name: Lint Rust Codebase runs-on: ubuntu-24.04 timeout-minutes: 10 - permissions: - contents: read - steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - name: Install stable Rust toolchain - uses: dtolnay/rust-toolchain@stable + uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable with: - toolchain: "1.85.0" + toolchain: "1.90.0" components: "clippy, rustfmt" - name: Cache cargo dependencies - uses: Swatinem/rust-cache@v2 + uses: Swatinem/rust-cache@e9d7b9f1e587218970671bac74e5511ed9506e6a # v2 - name: Ensure ripgrep shell: bash diff --git a/.github/workflows/php-lint.yml b/.github/workflows/php-lint.yml index 52b35d3..911a4df 100644 --- a/.github/workflows/php-lint.yml +++ b/.github/workflows/php-lint.yml @@ -19,8 +19,12 @@ jobs: environment: name: lint-php steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false + - uses: shivammathur/setup-php@d52fc211a0436f8839a587b3b5b8efddab4744ea # v2 + with: + php-version: '8.5' + coverage: none - name: Check syntax run: find frameworks/web-server/php -name '*.php' -print0 | xargs -0 -n1 php -l diff --git a/.github/workflows/python-lint.yml b/.github/workflows/python-lint.yml index 147a038..35d4452 100644 --- a/.github/workflows/python-lint.yml +++ b/.github/workflows/python-lint.yml @@ -19,13 +19,13 @@ jobs: environment: name: lint-python steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - - uses: actions/setup-python@v7 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 with: python-version: '3.13' - name: Install ruff - run: python -m pip install ruff==0.9.10 + run: python -m pip install ruff==0.17.1 - name: Check lint and format run: ruff check frameworks/web-server/python && ruff format --check frameworks/web-server/python diff --git a/.github/workflows/ruby-lint.yml b/.github/workflows/ruby-lint.yml index c51b045..7de3153 100644 --- a/.github/workflows/ruby-lint.yml +++ b/.github/workflows/ruby-lint.yml @@ -19,11 +19,11 @@ jobs: environment: name: lint-ruby steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - - uses: ruby/setup-ruby@v1 + - uses: ruby/setup-ruby@824c0e29fdfc7d689c191c27acc969d2d04b7dd8 # v1 with: - ruby-version: '3.3' + ruby-version: '3.4.11' - name: Check syntax run: find frameworks/web-server/ruby -name '*.rb' -print0 | xargs -0 -n1 ruby -c diff --git a/.github/workflows/rust-lint.yml b/.github/workflows/rust-lint.yml index da00680..d2c946d 100644 --- a/.github/workflows/rust-lint.yml +++ b/.github/workflows/rust-lint.yml @@ -19,13 +19,14 @@ jobs: environment: name: lint-rust steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@1.88.0 + - uses: dtolnay/rust-toolchain@b725eda525362e7cf34af9a7243cac1e3f01a7ec # stable with: + toolchain: "1.99.1" components: clippy, rustfmt - - uses: Swatinem/rust-cache@v2 + - uses: Swatinem/rust-cache@e9d7b9f1e587218970671bac74e5511ed9506e6a # v2 - name: Check formatting and clippy shell: bash run: | diff --git a/.github/workflows/zig-lint.yml b/.github/workflows/zig-lint.yml index 3f7888b..14f4820 100644 --- a/.github/workflows/zig-lint.yml +++ b/.github/workflows/zig-lint.yml @@ -19,10 +19,10 @@ jobs: environment: name: lint-zig steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: persist-credentials: false - - uses: mlugg/setup-zig@v2 + - uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2 with: version: 0.16.0 - name: Check formatting diff --git a/.gitignore b/.gitignore index ca82afe..f795409 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,18 @@ # Build artifacts target/ __pycache__/ +frameworks/web-server/go/*/*-benchmark +frameworks/web-server/rust/*/target/ +frameworks/web-server/csharp/*/bin/ +frameworks/web-server/csharp/*/obj/ +node_modules/ +.venv/ +vendor/ +frameworks/web-server/**/out +frameworks/web-server/**/server.log + +# Generated code knowledge graph +graphify-out/ # Operating System .DS_Store diff --git a/.graphifyignore b/.graphifyignore new file mode 100644 index 0000000..c2c013f --- /dev/null +++ b/.graphifyignore @@ -0,0 +1,7 @@ +.git/ +.agents/ +.direnv/ +.graphifyignore +graphify-out/ +results/ +target/ diff --git a/.opencode/opencode.json b/.opencode/opencode.json index bca55c3..22aaadc 100644 --- a/.opencode/opencode.json +++ b/.opencode/opencode.json @@ -1,4 +1,13 @@ { "$schema": "https://opencode.ai/config.json", - "subagent_depth": 1 + "subagent_depth": 1, + "mcp": { + "servers": { + "graphify": { + "type": "local", + "command": ["graphify-mcp", "graphify-out/graph.json"], + "cwd": "." + } + } + } } diff --git a/.opencode/plugins/graphify.js b/.opencode/plugins/graphify.js new file mode 100644 index 0000000..70ba849 --- /dev/null +++ b/.opencode/plugins/graphify.js @@ -0,0 +1,29 @@ +// graphify OpenCode plugin (V2) +// Injects a knowledge graph reminder before the first shell command when the graph exists. +// +// IMPORTANT: keep the reminder string free of backticks and $(...) constructs. +// The hook prepends `echo "" ; ` to the shell command; backticks +// inside the double-quoted echo trigger bash command substitution, which both +// corrupts tool output and silently executes the very graphify command we are +// only suggesting. Plain words render fine in opencode's TUI. +import { existsSync } from "fs"; +import { join } from "path"; + +export default { + id: "graphify", + async setup(ctx) { + let reminded = false; + + await ctx.shell.hook("create.before", (event) => { + if (reminded) return; + if (!existsSync(join(ctx.location.directory, "graphify-out", "graph.json"))) return; + + // ';' not '&&' — Windows PowerShell 5.1 rejects '&&' as a statement + // separator, breaking the first shell command of the session (#1646). + event.command = + 'echo "[graphify] knowledge graph at graphify-out/. For focused questions, run graphify query with your question (scoped subgraph, usually much smaller than GRAPH_REPORT.md) instead of grepping raw files. Read GRAPH_REPORT.md only for broad architecture context." ; ' + + event.command; + reminded = true; + }); + }, +}; diff --git a/AGENTS.md b/AGENTS.md index 05a9173..95a8492 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -6,7 +6,7 @@ You prioritize deterministic benchmarking, zero-allocation hot paths, data-orien ## Core Directives 1. **Target**: Build an industrial-grade, multi-language client and server framework benchmark platform rivaling TechEmpower and HttpArena. -2. **Language**: Rust (edition 2024 / 1.85+). Adhere to idiomatic, safe Rust with zero unneeded allocations on measurement paths. +2. **Language**: Rust (edition 2024 / 1.90+). Adhere to idiomatic, safe Rust with zero unneeded allocations on measurement paths. 3. **Dual-Libc Parity**: Ensure every benchmark suite and runtime evaluates both GNU C Library (`glibc`) and `musl` libc via Docker and Nix flakes. 4. **File Length Limit**: Strictly enforce a hard limit of at most 200 lines per file across all code, configuration, scripts, and documentation. 5. **Linux Coding Style**: Adhere to Linux kernel style principles: `snake_case` naming for files, functions, and variables; early returns; minimal nesting (max 3 levels); keyword spacing; and happy path unindented. diff --git a/CODEBASE.md b/CODEBASE.md index b559c68..7727960 100644 --- a/CODEBASE.md +++ b/CODEBASE.md @@ -36,6 +36,8 @@ framework-benchmark/ │ ├── architecture.md # Detailed engine architecture and lifecycle state machine │ ├── data_oriented_design.md# Data-oriented memory layout and SoA guidelines │ ├── separation_of_concerns.md# Architectural layer boundaries and decomposition +│ ├── small_host.md # Bounded 2 vCPU / 8 GB OpenStack test workflow +│ ├── graphify.md # Optional code knowledge graph for agent reviews │ ├── sub_agents.md # Antigravity, OpenCode, and Codex sub-agent workflows │ └── roadmap/ # Modularized implementation phases ├── scripts/ # Quality gate and verification scripts diff --git a/CODING_CONVENTION.md b/CODING_CONVENTION.md index f9c233d..2000358 100644 --- a/CODING_CONVENTION.md +++ b/CODING_CONVENTION.md @@ -1,6 +1,6 @@ # Framework Benchmark Coding Conventions -These conventions adapt the Linux Kernel Coding Style to Rust (edition 2024 / 1.85+), emphasizing pragmatism, performance, readability, and deterministic behavior. +These conventions adapt the Linux Kernel Coding Style to Rust (edition 2024 / 1.90+), emphasizing pragmatism, performance, readability, and deterministic behavior. ## 1. Core General Rules - **No Emojis**: Emojis are strictly forbidden anywhere in the repository, including code, comments, documentation, commit messages, and PRs. diff --git a/Cargo.lock b/Cargo.lock index 948a3e1..4dc94e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -16,23 +16,17 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" [[package]] name = "base64" -version = "0.21.7" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - -[[package]] -name = "bitflags" -version = "2.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "block-buffer" -version = "0.10.4" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" dependencies = [ - "generic-array", + "hybrid-array", ] [[package]] @@ -72,20 +66,26 @@ dependencies = [ "toml", ] +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "cpufeatures" -version = "0.2.17" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" dependencies = [ "libc", ] [[package]] name = "crc32fast" -version = "1.5.2" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055" dependencies = [ "cfg-if", ] @@ -107,39 +107,29 @@ checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] name = "crypto-common" -version = "0.1.7" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ - "generic-array", - "typenum", + "hybrid-array", ] [[package]] name = "digest" -version = "0.10.7" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer", + "const-oid", "crypto-common", ] [[package]] name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "errno" -version = "0.3.14" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" -dependencies = [ - "libc", - "windows-sys", -] +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" [[package]] name = "flate2" @@ -152,16 +142,6 @@ dependencies = [ "zlib-rs", ] -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "hashbrown" version = "0.17.1" @@ -170,9 +150,9 @@ checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "hdrhistogram" -version = "7.5.4" +version = "7.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" +checksum = "f49d1053f4708f0af3cf9fc5bffc7e68a914a3c45becb231c80068c9c3f78bea" dependencies = [ "base64", "byteorder", @@ -188,6 +168,15 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + [[package]] name = "indexmap" version = "2.14.2" @@ -206,15 +195,14 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "libc" -version = "0.2.190" +version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" [[package]] name = "load-gen" version = "0.1.0" dependencies = [ - "bytes", "common", "hdrhistogram", "httparse", @@ -222,27 +210,12 @@ dependencies = [ "tokio", ] -[[package]] -name = "lock_api" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" -dependencies = [ - "scopeguard", -] - [[package]] name = "memchr" version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - [[package]] name = "miniz_oxide" version = "0.9.1" @@ -266,12 +239,11 @@ dependencies = [ [[package]] name = "nom" -version = "7.1.3" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" dependencies = [ "memchr", - "minimal-lexical", ] [[package]] @@ -296,29 +268,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "parking_lot" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-link", -] - [[package]] name = "pin-project-lite" version = "0.2.17" @@ -343,15 +292,6 @@ dependencies = [ "proc-macro2", ] -[[package]] -name = "redox_syscall" -version = "0.5.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" -dependencies = [ - "bitflags", -] - [[package]] name = "reporter" version = "0.1.0" @@ -362,12 +302,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - [[package]] name = "serde" version = "1.0.229" @@ -413,46 +347,30 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.9" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +checksum = "4e7523beb55eece201a2356bee0bbca0d1ab466c14c07703b2e0ee6d42cb0c2c" dependencies = [ - "serde", + "serde_core", ] [[package]] name = "sha2" -version = "0.10.9" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" dependencies = [ "cfg-if", "cpufeatures", "digest", ] -[[package]] -name = "signal-hook-registry" -version = "1.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" -dependencies = [ - "errno", - "libc", -] - [[package]] name = "simd-adler32" version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" -[[package]] -name = "smallvec" -version = "1.16.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b" - [[package]] name = "socket2" version = "0.6.5" @@ -496,16 +414,14 @@ dependencies = [ [[package]] name = "tokio" -version = "1.53.2" +version = "1.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e95f91fcc7a621e8b030f6aa23c71fe9838ae2fb4d8118b75602a328f5144044" +checksum = "a0c168a474b95bfecfe908a65d4bdffc523144b649a74c09c286afb35d53e0bc" dependencies = [ "bytes", "libc", "mio", - "parking_lot", "pin-project-lite", - "signal-hook-registry", "socket2", "tokio-macros", "windows-sys", @@ -524,44 +440,42 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.23" +version = "1.1.8+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +checksum = "20489e00e4d8741d6be680764cc12e270655e375a20d1011e844a9c3379e678d" dependencies = [ - "serde", + "indexmap", + "serde_core", "serde_spanned", "toml_datetime", - "toml_edit", + "toml_parser", + "toml_writer", + "winnow", ] [[package]] name = "toml_datetime" -version = "0.6.11" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053" dependencies = [ - "serde", + "serde_core", ] [[package]] -name = "toml_edit" -version = "0.22.27" +name = "toml_parser" +version = "1.1.5+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" dependencies = [ - "indexmap", - "serde", - "serde_spanned", - "toml_datetime", - "toml_write", "winnow", ] [[package]] -name = "toml_write" -version = "0.1.2" +name = "toml_writer" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +checksum = "06bdbd8cfc056b8d2e2e85f29b56a3bdbecb527cef81eb39e3e7b98af4652770" [[package]] name = "typenum" @@ -575,12 +489,6 @@ version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -604,12 +512,9 @@ dependencies = [ [[package]] name = "winnow" -version = "0.7.15" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" -dependencies = [ - "memchr", -] +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" [[package]] name = "zlib-rs" diff --git a/Cargo.toml b/Cargo.toml index 6256029..26d9db0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,21 +11,26 @@ members = [ [workspace.package] version = "0.1.0" edition = "2024" -rust-version = "1.85" +rust-version = "1.90" authors = ["Farben "] license = "MIT" [workspace.dependencies] serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -toml = "0.8" +toml = "1.1" thiserror = "2.0" -tokio = { version = "1.43", features = ["full"] } -hdrhistogram = "7.5" -bytes = "1.10" -http = "1.2" +tokio = { version = "1.54", features = [ + "rt-multi-thread", + "macros", + "net", + "time", + "sync", + "io-util", +] } +hdrhistogram = "7.6" httparse = "1.10" -sha2 = "0.10" +sha2 = "0.11" [profile.release] opt-level = 3 diff --git a/README.md b/README.md index 3f69ccf..9059fd3 100644 --- a/README.md +++ b/README.md @@ -11,12 +11,16 @@ Inspired by [TechEmpower FrameworkBenchmarks](https://github.com/TechEmpower/Fra - **Extensible Framework Registry**: Declarative framework definitions in `frameworks/` using **Docker** and **Nix**. - **Native Rust Harness**: Ultra-low-overhead asynchronous load generator and Linux resource profiler (cgroups v2, `/proc`, peak RSS, CPU utilization). - **Publishable Artifacts**: Immutable JSON results include revision, container image ID, and exact traffic settings alongside category charts and complete Markdown tables. A labelled dedicated runner publishes reviewed results to this `README.md`. +- **Bounded Small-Host Profile**: A guarded 2 vCPU / 8 GB OpenStack wrapper, per-language smoke matrix, CPU pinning, memory limits, and reduced warmup for adapter validation without pretending small-host numbers are publishable. ## Project Roadmap and Architecture For the detailed architectural specification, component breakdown, and step-by-step developer and agent checklist, refer to: - Roadmap Summary: [TODO.md](TODO.md) - Architecture Overview: [docs/roadmap/overview.md](docs/roadmap/overview.md) +- Host Workflow: [docs/how_to_use.md](docs/how_to_use.md) +- Small OpenStack Hosts: [docs/small_host.md](docs/small_host.md) +- Code Knowledge Graph: [docs/graphify.md](docs/graphify.md) - Agent Persona and Directives: [AGENTS.md](AGENTS.md) - Coding Conventions: [CODING_CONVENTION.md](CODING_CONVENTION.md) diff --git a/TODO.md b/TODO.md index 80b4a1e..3a2e3d1 100644 --- a/TODO.md +++ b/TODO.md @@ -46,6 +46,8 @@ The Docker-based dual-libc baseline is implemented, but a result is publishable - Host-managed cgroups and CPU sets for complete collector attribution. [Completed] - Full-matrix execution on a labelled self-hosted runner. [Requires designated host] - Immutable image and framework input pinning before historical comparison. [Completed] +- Bounded 2 vCPU / 8 GB OpenStack profile and per-language smoke workflow. [Completed] +- Optional graphify code knowledge graph for agent-assisted reviews. [Completed] See [docs/how_to_use.md](docs/how_to_use.md) for the supported host workflow. diff --git a/crates/collector/src/aggregator.rs b/crates/collector/src/aggregator.rs index 82b1e1c..078c6aa 100644 --- a/crates/collector/src/aggregator.rs +++ b/crates/collector/src/aggregator.rs @@ -16,7 +16,7 @@ impl HarvestAggregator { self.peak_rss_bytes = rss_bytes; } self.rss_sum_bytes = self.rss_sum_bytes.saturating_add(rss_bytes); - self.sample_count += 1; + self.sample_count = self.sample_count.saturating_add(1); } pub fn finalize(self) -> ResourceUsage { diff --git a/crates/collector/src/ebpf.rs b/crates/collector/src/ebpf.rs index 29f5fbe..ea434ca 100644 --- a/crates/collector/src/ebpf.rs +++ b/crates/collector/src/ebpf.rs @@ -27,6 +27,8 @@ pub fn parse_ebpf_metrics(content: &str) -> EbpfMetrics { } pub fn page_cache_hit_ratio(metrics: EbpfMetrics) -> Option { - let total = metrics.page_cache_hits + metrics.page_cache_misses; + let total = metrics + .page_cache_hits + .saturating_add(metrics.page_cache_misses); (total > 0).then(|| metrics.page_cache_hits as f64 / total as f64) } diff --git a/crates/collector/src/lib.rs b/crates/collector/src/lib.rs index 853947f..4a3d1e0 100644 --- a/crates/collector/src/lib.rs +++ b/crates/collector/src/lib.rs @@ -13,6 +13,9 @@ pub use cgroups::{CgroupSample, parse_cgroup_cpu_stat, parse_cgroup_memory_stat} pub use ebpf::{EbpfMetrics, page_cache_hit_ratio, parse_ebpf_metrics}; pub use error::CollectorError; pub use perf::{PerfMode, parse_perf_event_paranoid, parse_perf_stat_value, select_perf_mode}; -pub use procfs::{ProcessSample, parse_stat_times, parse_statm, parse_status_switches}; +pub use procfs::{ + ProcessSample, parse_kernel_page_size, parse_stat_times, parse_statm, parse_status_switches, + resolve_page_size, select_page_size, +}; pub use rapl::{RaplZoneSnapshot, energy_delta_joules, energy_delta_uj, read_rapl_snapshot}; pub use sampler::BackgroundSampler; diff --git a/crates/collector/src/procfs.rs b/crates/collector/src/procfs.rs index eccd657..c9ccdc0 100644 --- a/crates/collector/src/procfs.rs +++ b/crates/collector/src/procfs.rs @@ -1,4 +1,9 @@ use crate::error::CollectorError; +use std::fs; + +const DEFAULT_PAGE_SIZE: u64 = 4096; +const SMAPS_ROLLUP_PATH: &str = "/proc/self/smaps_rollup"; +const SMAPS_PATH: &str = "/proc/self/smaps"; #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] pub struct ProcessSample { @@ -18,7 +23,33 @@ pub fn parse_statm(content: &str, page_size: u64) -> Result let pages: u64 = resident .parse() .map_err(|e| CollectorError::Parse(format!("invalid pages in statm: {e}")))?; - Ok(pages * page_size) + Ok(pages.saturating_mul(page_size)) +} + +pub fn parse_kernel_page_size(content: &str) -> Option { + content + .lines() + .filter_map(|line| line.strip_prefix("KernelPageSize:")) + .filter_map(|value| value.split_whitespace().next()?.parse::().ok()) + .find(|size| *size > 0) + .and_then(|kib| kib.checked_mul(1024)) +} + +pub fn select_page_size(rollup: Option<&str>, smaps: Option<&str>) -> u64 { + rollup + .and_then(parse_kernel_page_size) + .or_else(|| smaps.and_then(parse_kernel_page_size)) + .unwrap_or(DEFAULT_PAGE_SIZE) +} + +pub fn resolve_page_size() -> u64 { + let rollup = fs::read_to_string(SMAPS_ROLLUP_PATH).ok(); + let smaps = if rollup.as_deref().and_then(parse_kernel_page_size).is_some() { + None + } else { + fs::read_to_string(SMAPS_PATH).ok() + }; + select_page_size(rollup.as_deref(), smaps.as_deref()) } pub fn parse_status_switches(content: &str) -> (u64, u64) { @@ -41,3 +72,39 @@ pub fn parse_stat_times(content: &str) -> Option<(u64, u64)> { let kernel = fields.next()?.parse().ok()?; Some((user, kernel)) } + +#[cfg(test)] +mod tests { + use super::{parse_kernel_page_size, parse_statm, select_page_size}; + + #[test] + fn parse_kernel_page_size_reads_smaps_field() { + let content = "Rss: 1024 kB\nKernelPageSize: 16 kB\nMMUPageSize: 16 kB\n"; + assert_eq!(parse_kernel_page_size(content), Some(16_384)); + } + + #[test] + fn parse_kernel_page_size_rejects_missing_or_invalid_values() { + assert_eq!(parse_kernel_page_size("Rss: 1024 kB\n"), None); + assert_eq!(parse_kernel_page_size("KernelPageSize: invalid kB\n"), None); + assert_eq!(parse_kernel_page_size("KernelPageSize: 0 kB\n"), None); + let mixed = "KernelPageSize: 0 kB\nKernelPageSize: 4 kB\n"; + assert_eq!(parse_kernel_page_size(mixed), Some(4096)); + } + + #[test] + fn select_page_size_prefers_rollup_then_smaps() { + let rollup = "KernelPageSize: 16 kB\n"; + let smaps = "KernelPageSize: 64 kB\n"; + assert_eq!(select_page_size(Some(rollup), Some(smaps)), 16_384); + assert_eq!(select_page_size(None, Some(smaps)), 65_536); + assert_eq!(select_page_size(Some("invalid"), Some(smaps)), 65_536); + assert_eq!(select_page_size(None, None), 4096); + } + + #[test] + fn parse_statm_saturates_page_product() { + let content = "1 18446744073709551615 0 0 0 0 0\n"; + assert_eq!(parse_statm(content, 4096).ok(), Some(u64::MAX)); + } +} diff --git a/crates/collector/src/rapl.rs b/crates/collector/src/rapl.rs index 6ea63a2..9ec2176 100644 --- a/crates/collector/src/rapl.rs +++ b/crates/collector/src/rapl.rs @@ -34,6 +34,7 @@ fn read_rapl_directory( ) -> Result<(), CollectorError> { if path.join("energy_uj").is_file() { zones.push(read_rapl_zone(path)?); + return Ok(()); } for entry in fs::read_dir(path)? { let path = entry?.path(); @@ -62,7 +63,7 @@ pub fn energy_delta_joules(before: &[RaplZoneSnapshot], after: &[RaplZoneSnapsho energy_delta_uj(start.energy_uj, end.energy_uj, start.max_energy_range_uj) }) }) - .sum::() as f64 + .fold(0_u64, u64::saturating_add) as f64 / 1_000_000.0 } @@ -88,3 +89,31 @@ fn read_u64(path: &Path, name: &str) -> Result { .parse() .map_err(|error| CollectorError::Parse(format!("invalid RAPL {name}: {error}"))) } + +#[cfg(test)] +mod tests { + use super::read_rapl_snapshot; + use std::fs; + use std::path::Path; + + fn write_zone(path: &Path, label: &str, energy: u64) { + fs::create_dir_all(path).expect("create RAPL fixture"); + fs::write(path.join("name"), label).expect("write RAPL name"); + fs::write(path.join("energy_uj"), energy.to_string()).expect("write RAPL energy"); + fs::write(path.join("max_energy_range_uj"), "1000").expect("write RAPL range"); + } + + #[test] + fn nested_zones_do_not_double_count_package_energy() { + let root = std::env::temp_dir().join(format!("rapl-{}", std::process::id())); + let package = root.join("intel-rapl:0"); + write_zone(&package, "package-0", 100); + write_zone(&package.join("intel-rapl:0:0"), "core", 50); + + let zones = read_rapl_snapshot(&root).expect("read RAPL fixture"); + fs::remove_dir_all(&root).expect("remove RAPL fixture"); + + assert_eq!(zones.len(), 1); + assert_eq!(zones[0].energy_uj, 100); + } +} diff --git a/crates/collector/src/sampler.rs b/crates/collector/src/sampler.rs index bd78b81..9ae0174 100644 --- a/crates/collector/src/sampler.rs +++ b/crates/collector/src/sampler.rs @@ -1,17 +1,18 @@ use crate::aggregator::HarvestAggregator; -use crate::cgroups::{CgroupFiles, open_cgroup_files, sample_cgroup_files}; use crate::metrics::{RuntimeCounters, apply_runtime_counters}; -use crate::procfs::{parse_statm, parse_status_switches}; use crate::rapl::{RaplZoneSnapshot, energy_delta_joules, read_rapl_snapshot}; use common::result::ResourceUsage; -use std::fs::File; -use std::io::{Read, Seek, SeekFrom}; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; use tokio::sync::Mutex; use tokio::task::JoinHandle; +use tokio::time::MissedTickBehavior; + +mod sources; + +use sources::{CounterPaths, counter_paths, open_sources, read_counters, sample_memory}; const CGROUP_INTERVAL: Duration = Duration::from_millis(100); @@ -26,18 +27,6 @@ pub struct BackgroundSampler { started: Instant, } -#[derive(Debug)] -struct Sources { - cgroups: Option, - statm: Option, -} - -#[derive(Debug, Default)] -struct CounterPaths { - cpu_stat: Option, - status: Option, -} - impl BackgroundSampler { pub fn spawn_target( target_pid: Option, @@ -54,12 +43,11 @@ impl BackgroundSampler { let interval = cgroup_dir.map_or(interval, |_| CGROUP_INTERVAL); let handle = tokio::spawn(async move { let mut ticker = tokio::time::interval(interval); + ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); while run_flag.load(Ordering::Relaxed) { ticker.tick().await; - let (memory_bytes, peak_bytes) = sample_memory(&mut sources); - let mut agg = agg_lock.lock().await; - agg.record_sample(memory_bytes); - agg.peak_rss_bytes = agg.peak_rss_bytes.max(peak_bytes); + let (memory_bytes, _) = sample_memory(&mut sources); + agg_lock.lock().await.record_sample(memory_bytes); } }); Self { @@ -80,6 +68,7 @@ impl BackgroundSampler { let agg_lock = Arc::clone(&aggregator); let handle = tokio::spawn(async move { let mut ticker = tokio::time::interval(interval); + ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); while run_flag.load(Ordering::Relaxed) { ticker.tick().await; agg_lock.lock().await.record_sample(static_rss); @@ -97,12 +86,9 @@ impl BackgroundSampler { } pub async fn stop(mut self) -> ResourceUsage { - self.running.store(false, Ordering::Relaxed); - if let Some(handle) = self.handle.take() { - let _ = handle.await; - } + self.stop_task().await; let mut usage = self.aggregator.lock().await.clone().finalize(); - let elapsed_usec = self.started.elapsed().as_micros() as u64; + let elapsed_usec = u64::try_from(self.started.elapsed().as_micros()).unwrap_or(u64::MAX); apply_runtime_counters( &mut usage, self.counter_start, @@ -115,65 +101,60 @@ impl BackgroundSampler { } usage } -} -fn counter_paths(target_pid: Option, cgroup_dir: Option<&Path>) -> CounterPaths { - CounterPaths { - cpu_stat: cgroup_dir.map(|path| path.join("cpu.stat")), - status: target_pid.map(|pid| PathBuf::from(format!("/proc/{pid}/status"))), + async fn stop_task(&mut self) { + self.running.store(false, Ordering::Relaxed); + let Some(handle) = self.handle.take() else { + return; + }; + handle.abort(); + let _ = handle.await; } } -fn read_counters(paths: &CounterPaths) -> RuntimeCounters { - let (user_usec, kernel_usec) = paths - .cpu_stat - .as_ref() - .and_then(|path| std::fs::read_to_string(path).ok()) - .map(|content| crate::cgroups::parse_cgroup_cpu_stat(&content)) - .unwrap_or_default(); - let context_switches = paths - .status - .as_ref() - .and_then(|path| std::fs::read_to_string(path).ok()) - .map(|content| parse_status_switches(&content)) - .map(|(voluntary, involuntary)| voluntary.saturating_add(involuntary)) - .unwrap_or_default(); - RuntimeCounters { - user_usec, - kernel_usec, - context_switches, +impl Drop for BackgroundSampler { + fn drop(&mut self) { + self.running.store(false, Ordering::Relaxed); + if let Some(handle) = self.handle.take() { + handle.abort(); + } } } -fn open_sources(target_pid: Option, cgroup_dir: Option<&Path>) -> Sources { - let pid = target_pid.map_or_else(|| "self".to_owned(), |pid| pid.to_string()); - Sources { - cgroups: cgroup_dir.map(open_cgroup_files), - statm: File::open(format!("/proc/{pid}/statm")).ok(), - } -} +#[cfg(test)] +mod tests { + use super::BackgroundSampler; + use std::sync::Arc; + use std::sync::atomic::Ordering; + use std::time::Duration; -fn sample_memory(sources: &mut Sources) -> (u64, u64) { - let process_rss = read_statm(&mut sources.statm); - let cgroup = sources.cgroups.as_mut().map(sample_cgroup_files); - let current = cgroup.map_or(0, |sample| sample.current_bytes); - let peak = cgroup.map_or(0, |sample| sample.peak_bytes); - (process_rss.max(current), process_rss.max(peak)) -} + #[tokio::test] + async fn drop_aborts_sampling_task() { + let sampler = BackgroundSampler::spawn_simulated(Duration::from_secs(60), 1024); + let running = Arc::clone(&sampler.running); + let abort = sampler + .handle + .as_ref() + .map(tokio::task::JoinHandle::abort_handle); + drop(sampler); + + assert!(!running.load(Ordering::Relaxed)); + let abort = abort.expect("sampler task handle"); + let wait = async { + while !abort.is_finished() { + tokio::task::yield_now().await; + } + }; + tokio::time::timeout(Duration::from_secs(1), wait) + .await + .expect("aborted sampler task should finish"); + } -fn read_statm(file: &mut Option) -> u64 { - let Some(file) = file.as_mut() else { - return 0; - }; - let mut buffer = [0; 128]; - if file.seek(SeekFrom::Start(0)).is_err() { - return 0; + #[tokio::test] + async fn stop_task_is_idempotent() { + let mut sampler = BackgroundSampler::spawn_simulated(Duration::from_secs(60), 1024); + sampler.stop_task().await; + sampler.stop_task().await; + assert!(!sampler.running.load(Ordering::Relaxed)); } - let Ok(count) = file.read(&mut buffer) else { - return 0; - }; - let Ok(content) = std::str::from_utf8(&buffer[..count]) else { - return 0; - }; - parse_statm(content, 4096).unwrap_or(0) } diff --git a/crates/collector/src/sampler/sources.rs b/crates/collector/src/sampler/sources.rs new file mode 100644 index 0000000..4de3c73 --- /dev/null +++ b/crates/collector/src/sampler/sources.rs @@ -0,0 +1,93 @@ +use crate::cgroups::{CgroupFiles, open_cgroup_files, parse_cgroup_cpu_stat, sample_cgroup_files}; +use crate::metrics::RuntimeCounters; +use crate::procfs::{parse_stat_times, parse_statm, parse_status_switches, resolve_page_size}; +use std::fs::File; +use std::io::{Read, Seek, SeekFrom}; +use std::path::{Path, PathBuf}; + +#[derive(Debug)] +pub(super) struct Sources { + cgroups: Option, + statm: Option, + page_size: u64, +} + +#[derive(Debug, Default)] +pub(super) struct CounterPaths { + cpu_stat: Option, + stat: Option, + status: Option, +} + +pub(super) fn counter_paths(target_pid: Option, cgroup_dir: Option<&Path>) -> CounterPaths { + CounterPaths { + cpu_stat: cgroup_dir.map(|path| path.join("cpu.stat")), + stat: target_pid.map(|pid| PathBuf::from(format!("/proc/{pid}/stat"))), + status: target_pid.map(|pid| PathBuf::from(format!("/proc/{pid}/status"))), + } +} + +pub(super) fn read_counters(paths: &CounterPaths) -> RuntimeCounters { + let (user_usec, kernel_usec) = if let Some(content) = paths + .cpu_stat + .as_ref() + .and_then(|path| std::fs::read_to_string(path).ok()) + { + parse_cgroup_cpu_stat(&content) + } else { + paths + .stat + .as_ref() + .and_then(|path| std::fs::read_to_string(path).ok()) + .and_then(|content| parse_stat_times(&content)) + // Linux USER_HZ is 100 on the supported x86_64 and aarch64 hosts. + .map(|(user, kernel)| (user.saturating_mul(10_000), kernel.saturating_mul(10_000))) + .unwrap_or_default() + }; + let context_switches = paths + .status + .as_ref() + .and_then(|path| std::fs::read_to_string(path).ok()) + .map(|content| parse_status_switches(&content)) + .map(|(voluntary, involuntary)| voluntary.saturating_add(involuntary)) + .unwrap_or_default(); + RuntimeCounters { + user_usec, + kernel_usec, + context_switches, + } +} + +pub(super) fn open_sources(target_pid: Option, cgroup_dir: Option<&Path>) -> Sources { + let pid = target_pid.map_or_else(|| "self".to_owned(), |pid| pid.to_string()); + Sources { + cgroups: cgroup_dir.map(open_cgroup_files), + statm: File::open(format!("/proc/{pid}/statm")).ok(), + page_size: resolve_page_size(), + } +} + +pub(super) fn sample_memory(sources: &mut Sources) -> (u64, u64) { + let process_rss = read_statm(&mut sources.statm, sources.page_size); + let cgroup = sources.cgroups.as_mut().map(sample_cgroup_files); + let current = cgroup.map_or(0, |sample| sample.current_bytes); + let peak = cgroup.map_or(0, |sample| sample.peak_bytes); + (process_rss.max(current), process_rss.max(peak)) +} + +fn read_statm(file: &mut Option, page_size: u64) -> u64 { + let Some(file) = file.as_mut() else { + return 0; + }; + let mut buffer = [0; 128]; + if file.seek(SeekFrom::Start(0)).is_err() { + return 0; + } + let Ok(count) = file.read(&mut buffer) else { + return 0; + }; + let Ok(content) = std::str::from_utf8(&buffer[..count]) else { + return 0; + }; + parse_statm(content, page_size).unwrap_or(0) +} diff --git a/crates/collector/tests/collector_test.rs b/crates/collector/tests/collector_test.rs index e325aeb..2a44a02 100644 --- a/crates/collector/tests/collector_test.rs +++ b/crates/collector/tests/collector_test.rs @@ -107,3 +107,24 @@ fn test_runtime_counter_delta() { assert!((usage.cpu_kernel_pct - 25.0).abs() < f32::EPSILON); assert_eq!(usage.context_switches, 12); } + +#[test] +fn test_page_cache_hit_ratio_saturates_totals() { + let metrics = EbpfMetrics { + page_cache_hits: u64::MAX, + page_cache_misses: 1, + ..EbpfMetrics::default() + }; + let ratio = page_cache_hit_ratio(metrics).expect("ratio for saturated counters"); + assert!((ratio - 1.0).abs() < f64::EPSILON); +} + +#[test] +fn test_harvest_aggregator_sample_count_saturates() { + let mut agg = HarvestAggregator { + sample_count: u64::MAX, + ..HarvestAggregator::default() + }; + agg.record_sample(1); + assert_eq!(agg.sample_count, u64::MAX); +} diff --git a/crates/collector/tests/sampler_test.rs b/crates/collector/tests/sampler_test.rs index 2a5fad7..a697b19 100644 --- a/crates/collector/tests/sampler_test.rs +++ b/crates/collector/tests/sampler_test.rs @@ -34,5 +34,5 @@ async fn target_sampler_should_combine_process_and_cgroup_memory() { let usage = sampler.stop().await; fs::remove_dir_all(&path).expect("remove cgroup fixture directory"); - assert!(usage.peak_rss_bytes == peak && usage.avg_rss_bytes > 1024); + assert!(usage.peak_rss_bytes < peak && usage.avg_rss_bytes > 1024); } diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index cc0516a..12e154f 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -1,7 +1,10 @@ pub mod error; pub mod manifest; pub mod result; +mod validation; +mod validation_build; pub use error::CommonError; -pub use manifest::{FrameworkManifest, parse_manifest, validate_manifest}; +pub use manifest::{FrameworkManifest, parse_manifest}; pub use result::{LibcVariant, RunResult, deserialize_result, serialize_result_pretty}; +pub use validation::validate_manifest; diff --git a/crates/common/src/manifest.rs b/crates/common/src/manifest.rs index b42251c..717c822 100644 --- a/crates/common/src/manifest.rs +++ b/crates/common/src/manifest.rs @@ -90,66 +90,6 @@ pub struct FrameworkManifest { pub fn parse_manifest(toml_str: &str) -> Result { let manifest: FrameworkManifest = toml::from_str(toml_str)?; - validate_manifest(&manifest)?; + crate::validation::validate_manifest(&manifest)?; Ok(manifest) } - -pub fn validate_manifest(m: &FrameworkManifest) -> Result<(), CommonError> { - if m.framework.name.trim().is_empty() { - return Err(CommonError::Validation( - "framework name cannot be empty".into(), - )); - } - if m.framework.language.trim().is_empty() { - return Err(CommonError::Validation( - "framework language cannot be empty".into(), - )); - } - if m.runtime.port == 0 { - return Err(CommonError::Validation("runtime port cannot be 0".into())); - } - if !m.runtime.healthcheck_path.starts_with('/') { - return Err(CommonError::Validation( - "healthcheck_path must start with '/'".into(), - )); - } - if m.runtime - .websocket_path - .as_deref() - .is_some_and(|path| !path.starts_with('/')) - { - return Err(CommonError::Validation( - "websocket_path must start with '/'".into(), - )); - } - if m.build.engine != "docker" && m.build.engine != "nix" { - return Err(CommonError::Validation( - "build engine must be 'docker' or 'nix'".into(), - )); - } - if m.build.engine == "docker" { - let Some(docker) = &m.build.docker else { - return Err(CommonError::Validation( - "docker engine requires docker config".into(), - )); - }; - if docker.dockerfile_glibc.is_none() || docker.dockerfile_musl.is_none() { - return Err(CommonError::Validation( - "docker engine requires glibc and musl Dockerfiles".into(), - )); - } - } - if m.build.engine == "nix" { - let Some(nix) = &m.build.nix else { - return Err(CommonError::Validation( - "nix engine requires nix config".into(), - )); - }; - if nix.flake_attr_glibc.is_none() || nix.flake_attr_musl.is_none() { - return Err(CommonError::Validation( - "nix engine requires glibc and musl flake attributes".into(), - )); - } - } - Ok(()) -} diff --git a/crates/common/src/result.rs b/crates/common/src/result.rs index d69a514..4a1b041 100644 --- a/crates/common/src/result.rs +++ b/crates/common/src/result.rs @@ -32,7 +32,7 @@ pub struct LatencyDistribution { pub std_dev_us: f64, } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] pub struct ResourceUsage { pub peak_rss_bytes: u64, pub avg_rss_bytes: u64, diff --git a/crates/common/src/validation.rs b/crates/common/src/validation.rs new file mode 100644 index 0000000..63bd6dc --- /dev/null +++ b/crates/common/src/validation.rs @@ -0,0 +1,156 @@ +use crate::error::CommonError; +use crate::manifest::FrameworkManifest; + +const MAX_NAME_BYTES: usize = 64; +const MAX_VERSION_BYTES: usize = 64; +const MAX_PATH_BYTES: usize = 256; +const MAX_ENV_BYTES: usize = 1024; + +pub fn validate_manifest(manifest: &FrameworkManifest) -> Result<(), CommonError> { + validate_metadata(manifest)?; + validate_runtime(manifest)?; + crate::validation_build::validate_build(manifest) +} + +fn validate_metadata(manifest: &FrameworkManifest) -> Result<(), CommonError> { + let framework = &manifest.framework; + if framework.name.trim().is_empty() { + return Err(CommonError::Validation( + "framework name cannot be empty".into(), + )); + } + if !valid_name(&framework.name) { + return Err(CommonError::Validation( + "framework name must contain only lowercase ASCII letters, digits, '-' or '_'".into(), + )); + } + if !valid_version(&framework.version) { + return Err(CommonError::Validation( + "framework version contains invalid characters".into(), + )); + } + if framework.language.trim().is_empty() { + return Err(CommonError::Validation( + "framework language cannot be empty".into(), + )); + } + if !valid_language(&framework.language) { + return Err(CommonError::Validation( + "framework language contains invalid characters".into(), + )); + } + if !valid_text(&framework.language_version, 64) { + return Err(CommonError::Validation( + "language_version contains invalid characters".into(), + )); + } + if !valid_name(&framework.category) { + return Err(CommonError::Validation( + "category must contain only lowercase ASCII letters, digits, '-' or '_'".into(), + )); + } + if framework + .tier + .as_deref() + .is_some_and(|tier| !valid_name(tier)) + { + return Err(CommonError::Validation( + "tier contains invalid characters".into(), + )); + } + if let Some(website) = framework.website.as_deref() + && (!website.starts_with("https://") || !valid_text(website, MAX_PATH_BYTES)) + { + return Err(CommonError::Validation( + "website must be an https URL without whitespace".into(), + )); + } + Ok(()) +} + +fn validate_runtime(manifest: &FrameworkManifest) -> Result<(), CommonError> { + let runtime = &manifest.runtime; + if runtime.port == 0 { + return Err(CommonError::Validation("runtime port cannot be 0".into())); + } + if !valid_endpoint_path(&runtime.healthcheck_path) { + return Err(CommonError::Validation( + "healthcheck_path must start with '/' and contain no whitespace".into(), + )); + } + if runtime + .websocket_path + .as_deref() + .is_some_and(|path| !valid_endpoint_path(path)) + { + return Err(CommonError::Validation( + "websocket_path must start with '/' and contain no whitespace".into(), + )); + } + if !(1..=600).contains(&runtime.startup_timeout_seconds) { + return Err(CommonError::Validation( + "startup_timeout_seconds must be between 1 and 600".into(), + )); + } + if runtime + .shutdown_signal + .as_deref() + .is_some_and(|signal| !valid_text(signal, 32)) + { + return Err(CommonError::Validation( + "shutdown_signal contains invalid characters".into(), + )); + } + for (key, value) in &runtime.environment { + if !valid_env_key(key) || !valid_text(value, MAX_ENV_BYTES) { + return Err(CommonError::Validation(format!( + "runtime environment entry is invalid: {key}" + ))); + } + } + Ok(()) +} + +pub(crate) fn valid_text(value: &str, max: usize) -> bool { + !value.is_empty() && value.len() <= max && !value.bytes().any(|byte| byte.is_ascii_control()) +} + +fn valid_name(value: &str) -> bool { + !value.is_empty() + && value.len() <= MAX_NAME_BYTES + && value.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'-' | b'_') + }) +} + +fn valid_version(value: &str) -> bool { + !value.is_empty() + && value.len() <= MAX_VERSION_BYTES + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b'+')) +} + +fn valid_language(value: &str) -> bool { + !value.is_empty() + && value.len() <= 32 + && value.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'+' | b'#' | b'-') + }) +} + +fn valid_endpoint_path(value: &str) -> bool { + value.starts_with('/') + && valid_text(value, MAX_PATH_BYTES) + && !value.bytes().any(|byte| byte.is_ascii_whitespace()) +} + +pub(crate) fn valid_env_key(value: &str) -> bool { + let mut bytes = value.bytes(); + let Some(first) = bytes.next() else { + return false; + }; + (first.is_ascii_alphabetic() || first == b'_') + && value.len() <= 128 + && bytes.all(|byte| byte.is_ascii_alphanumeric() || byte == b'_') +} diff --git a/crates/common/src/validation_build.rs b/crates/common/src/validation_build.rs new file mode 100644 index 0000000..5f97aeb --- /dev/null +++ b/crates/common/src/validation_build.rs @@ -0,0 +1,95 @@ +use crate::error::CommonError; +use crate::manifest::{FrameworkManifest, NixBuildConfig}; +use crate::validation::{valid_env_key, valid_text}; +use std::path::{Component, Path}; + +const MAX_PATH_BYTES: usize = 256; +const MAX_ENV_BYTES: usize = 1024; + +pub(crate) fn validate_build(manifest: &FrameworkManifest) -> Result<(), CommonError> { + let build = &manifest.build; + if build.engine != "docker" && build.engine != "nix" { + return Err(CommonError::Validation( + "build engine must be 'docker' or 'nix'".into(), + )); + } + if build.engine == "docker" { + validate_docker(manifest)?; + } + if build.engine == "nix" { + let Some(nix) = &build.nix else { + return Err(CommonError::Validation( + "nix engine requires nix config".into(), + )); + }; + validate_nix(nix)?; + } + Ok(()) +} + +fn validate_docker(manifest: &FrameworkManifest) -> Result<(), CommonError> { + let Some(docker) = &manifest.build.docker else { + return Err(CommonError::Validation( + "docker engine requires docker config".into(), + )); + }; + if docker.dockerfile_glibc.is_none() || docker.dockerfile_musl.is_none() { + return Err(CommonError::Validation( + "docker engine requires glibc and musl Dockerfiles".into(), + )); + } + for path in [&docker.dockerfile_glibc, &docker.dockerfile_musl] + .into_iter() + .flatten() + { + if !valid_relative_path(path) { + return Err(CommonError::Validation( + "dockerfile path must be a relative file name".into(), + )); + } + } + for (key, value) in &docker.build_args { + if !valid_env_key(key) || !valid_text(value, MAX_ENV_BYTES) { + return Err(CommonError::Validation(format!( + "docker build argument is invalid: {key}" + ))); + } + } + Ok(()) +} + +fn validate_nix(nix: &NixBuildConfig) -> Result<(), CommonError> { + if nix.flake_attr_glibc.is_none() || nix.flake_attr_musl.is_none() { + return Err(CommonError::Validation( + "nix engine requires glibc and musl flake attributes".into(), + )); + } + for attr in [&nix.flake_attr_glibc, &nix.flake_attr_musl] + .into_iter() + .flatten() + { + if !valid_nix_attr(attr) { + return Err(CommonError::Validation( + "nix flake attribute is invalid".into(), + )); + } + } + Ok(()) +} + +fn valid_relative_path(value: &str) -> bool { + !value.is_empty() + && value.len() <= MAX_PATH_BYTES + && Path::new(value) + .components() + .all(|component| matches!(component, Component::Normal(_))) +} + +fn valid_nix_attr(value: &str) -> bool { + !value.is_empty() + && value.len() <= MAX_PATH_BYTES + && !value.contains("..") + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b'/')) +} diff --git a/crates/common/tests/manifest_test.rs b/crates/common/tests/manifest_test.rs index fc96803..83684ae 100644 --- a/crates/common/tests/manifest_test.rs +++ b/crates/common/tests/manifest_test.rs @@ -61,6 +61,39 @@ fn test_reject_empty_name() { assert!(err.to_string().contains("framework name cannot be empty")); } +#[test] +fn test_reject_path_traversal_and_injection_values() { + let unsafe_category = + VALID_TOML.replace("category = \"web-server\"", "category = \"../escape\""); + let unsafe_dockerfile = VALID_TOML.replace( + "dockerfile_glibc = \"Dockerfile.glibc\"", + "dockerfile_glibc = \"../Dockerfile.glibc\"", + ); + let unsafe_healthcheck = VALID_TOML.replace( + "healthcheck_path = \"/healthz\"", + "healthcheck_path = \"/health z\"", + ); + assert!(parse_manifest(&unsafe_category).is_err()); + assert!(parse_manifest(&unsafe_dockerfile).is_err()); + assert!(parse_manifest(&unsafe_healthcheck).is_err()); +} + +#[test] +fn test_accepts_current_registry_metadata_forms() { + let manifest = parse_manifest( + &VALID_TOML + .replace("name = \"axum\"", "name = \"aspnet-minimal\"") + .replace("version = \"0.8\"", "version = \"v20.69.0\"") + .replace("language = \"rust\"", "language = \"csharp\"") + .replace( + "language_version = \"1.85\"", + "language_version = \".net 9\"", + ), + ) + .expect("registry metadata form should validate"); + assert_eq!(manifest.framework.name, "aspnet-minimal"); +} + #[test] fn test_reject_invalid_engine() { let invalid_toml = VALID_TOML.replace("engine = \"docker\"", "engine = \"kubernetes\""); diff --git a/crates/load-gen/Cargo.toml b/crates/load-gen/Cargo.toml index 955a5a1..7300ca4 100644 --- a/crates/load-gen/Cargo.toml +++ b/crates/load-gen/Cargo.toml @@ -9,7 +9,6 @@ license.workspace = true common = { path = "../common" } tokio = { workspace = true } hdrhistogram = { workspace = true } -bytes = { workspace = true } httparse = { workspace = true } thiserror = { workspace = true } diff --git a/crates/load-gen/src/autobahn/builder.rs b/crates/load-gen/src/autobahn/builder.rs index caf7d64..8823bf2 100644 --- a/crates/load-gen/src/autobahn/builder.rs +++ b/crates/load-gen/src/autobahn/builder.rs @@ -10,7 +10,7 @@ impl FrameBuilder { mask: [u8; 4], fin: bool, ) -> Vec { - let mut frame = Vec::with_capacity(payload.len() + 14); + let mut frame = Vec::with_capacity(payload.len().saturating_add(14)); let mut b0 = opcode as u8; if fin { diff --git a/crates/load-gen/src/autobahn/frame.rs b/crates/load-gen/src/autobahn/frame.rs index 39f93da..17d36e3 100644 --- a/crates/load-gen/src/autobahn/frame.rs +++ b/crates/load-gen/src/autobahn/frame.rs @@ -147,7 +147,9 @@ fn validate_control_frame(header: &FrameHeader) -> Result<(), LoadGenError> { } pub fn apply_mask(payload: &mut [u8], mask: [u8; 4]) { - for (i, byte) in payload.iter_mut().enumerate() { - *byte ^= mask[i % 4]; + for chunk in payload.chunks_mut(4) { + for (offset, byte) in chunk.iter_mut().enumerate() { + *byte ^= mask[offset]; + } } } diff --git a/crates/load-gen/src/autobahn/fuzz.rs b/crates/load-gen/src/autobahn/fuzz.rs index 1e7657a..9ab0d23 100644 --- a/crates/load-gen/src/autobahn/fuzz.rs +++ b/crates/load-gen/src/autobahn/fuzz.rs @@ -15,7 +15,7 @@ pub struct FuzzInjector; impl FuzzInjector { pub fn create_unmasked_frame(payload: &[u8]) -> Vec { - let mut frame = Vec::with_capacity(payload.len() + 10); + let mut frame = Vec::with_capacity(payload.len().saturating_add(10)); frame.push(0x81); let len = payload.len(); if len <= 125 { diff --git a/crates/load-gen/src/autobahn/runner.rs b/crates/load-gen/src/autobahn/runner.rs index f77cf22..6b332b1 100644 --- a/crates/load-gen/src/autobahn/runner.rs +++ b/crates/load-gen/src/autobahn/runner.rs @@ -55,8 +55,7 @@ impl AutobahnSuiteRunner { }; let _ = stream.set_nodelay(true); if let Err(error) = perform_handshake(&mut stream, addr, path, deadline).await { - stats.record_error(); - return Err(error); + return finish_or_error(error, deadline, stats); } let msg = "framework-benchmark-echo"; @@ -67,16 +66,12 @@ impl AutobahnSuiteRunner { while Instant::now() < deadline { let start = Instant::now(); if let Err(error) = write_until(&mut stream, &frame, deadline).await { - stats.record_error(); - return Err(error); + return finish_or_error(error, deadline, stats); } let frame_len = match read_echo_frame(&mut stream, msg.as_bytes(), &mut buf, deadline).await { Ok(frame_len) => frame_len, - Err(error) => { - stats.record_error(); - return Err(error); - } + Err(error) => return finish_or_error(error, deadline, stats), }; let elapsed_us = start.elapsed().as_micros() as u64; recorder.record_micros(elapsed_us); @@ -113,6 +108,18 @@ impl AutobahnSuiteRunner { } } +fn finish_or_error( + error: LoadGenError, + deadline: Instant, + stats: &mut WorkerStats, +) -> Result<(), LoadGenError> { + if Instant::now() >= deadline { + return Ok(()); + } + stats.record_error(); + Err(error) +} + async fn read_echo_frame( stream: &mut TcpStream, expected: &[u8], @@ -138,3 +145,29 @@ async fn read_echo_frame( used += read; } } + +#[cfg(test)] +mod tests { + use super::finish_or_error; + use crate::error::LoadGenError; + use crate::stats::WorkerStats; + use std::time::{Duration, Instant}; + + #[test] + fn expiry_is_not_a_server_error() { + let mut stats = WorkerStats::default(); + let deadline = Instant::now() + .checked_sub(Duration::from_millis(1)) + .expect("test deadline"); + assert!(finish_or_error(LoadGenError::Timeout, deadline, &mut stats).is_ok()); + assert_eq!(stats.errors, 0); + } + + #[test] + fn early_failure_is_recorded() { + let mut stats = WorkerStats::default(); + let deadline = Instant::now() + Duration::from_secs(1); + assert!(finish_or_error(LoadGenError::Timeout, deadline, &mut stats).is_err()); + assert_eq!(stats.errors, 1); + } +} diff --git a/crates/load-gen/src/client.rs b/crates/load-gen/src/client.rs index 1091c91..9e788e0 100644 --- a/crates/load-gen/src/client.rs +++ b/crates/load-gen/src/client.rs @@ -12,6 +12,7 @@ struct WorkerTask<'a> { pipeline_depth: usize, connection_mode: ConnectionMode, expected_interval: Option, + next_issue: Instant, recorder: &'a mut LatencyRecorder, stats: &'a mut WorkerStats, } @@ -51,27 +52,30 @@ pub(crate) async fn run_connection_worker_with_buffers( connection_mode, expected_interval: (expected_interval_us > 0) .then(|| Duration::from_micros(expected_interval_us)), + next_issue: Instant::now(), recorder, stats, }; while Instant::now() < task.deadline { let stream = connect_until(target_addr, task.deadline).await; let Ok(mut stream) = stream else { + if Instant::now() >= task.deadline { + return Ok(()); + } task.stats.record_error(); retry_after_error(task.deadline).await; continue; }; let _ = stream.set_nodelay(true); - if process_connection_loop( + let result = process_connection_loop( &mut stream, request_bytes, &mut buffers.read_buf, &mut buffers.starts, &mut task, ) - .await - .is_err() - { + .await; + if result.is_err() && Instant::now() < task.deadline { task.stats.record_error(); } } @@ -84,18 +88,22 @@ async fn process_connection_loop( starts: &mut Vec, task: &mut WorkerTask<'_>, ) -> Result<(), LoadGenError> { - let mut next_issue = Instant::now(); let mut used = 0; while Instant::now() < task.deadline { + if used != 0 { + return Err(LoadGenError::Protocol( + "unexpected bytes after response batch".into(), + )); + } starts.clear(); for _ in 0..task.pipeline_depth { - if !wait_until(next_issue, task.deadline).await { + if !wait_until(task.next_issue, task.deadline).await { return Ok(()); } let start = Instant::now(); write_until(stream, request_bytes, task.deadline).await?; starts.push(start); - next_issue = next_issue_at(task.expected_interval, next_issue, task.deadline); + task.next_issue = next_issue_at(task.expected_interval, task.next_issue, task.deadline); } used = read_batch(stream, read_buf, used, starts, task).await?; if task.connection_mode == ConnectionMode::Churn { diff --git a/crates/load-gen/src/engine.rs b/crates/load-gen/src/engine.rs index c3b8fb8..5628fac 100644 --- a/crates/load-gen/src/engine.rs +++ b/crates/load-gen/src/engine.rs @@ -39,14 +39,16 @@ pub async fn execute_benchmark( _ => 0, }; let mut worker_buffers = Vec::with_capacity(concurrency); + let mut recorders = Vec::with_capacity(concurrency); for _ in 0..concurrency { worker_buffers.push(preallocate_worker_buffers(plan.pipeline_depth)); + recorders.push(LatencyRecorder::new()?); } let start_instant = Instant::now(); let deadline = start_instant + Duration::from_secs(plan.duration_secs); let mut handles = Vec::with_capacity(concurrency); - for buffers in worker_buffers { + for (buffers, recorder) in worker_buffers.into_iter().zip(recorders) { let addr = Arc::clone(&target_addr); let bytes = Arc::clone(&req_bytes); let depth = plan.pipeline_depth; @@ -54,7 +56,7 @@ pub async fn execute_benchmark( let interval = expected_interval_us; let handle = tokio::spawn(async move { - let mut recorder = LatencyRecorder::default(); + let mut recorder = recorder; let mut stats = WorkerStats::default(); let result = run_connection_worker_with_buffers( &addr, @@ -68,7 +70,7 @@ pub async fn execute_benchmark( buffers, ) .await; - if result.is_err() { + if result.is_err() && Instant::now() < deadline { stats.record_error(); } (recorder, stats) @@ -76,7 +78,7 @@ pub async fn execute_benchmark( handles.push(handle); } - let mut final_recorder = LatencyRecorder::default(); + let mut final_recorder = LatencyRecorder::new()?; let mut final_stats = WorkerStats::default(); for handle in handles { @@ -89,6 +91,12 @@ pub async fn execute_benchmark( } } + if final_stats.requests_completed == 0 { + return Err(LoadGenError::Protocol( + "load generator completed no responses".into(), + )); + } + let actual_duration_secs = start_instant.elapsed().as_secs_f64().max(0.001); let rps = final_stats.requests_completed as f64 / actual_duration_secs; let mb_per_sec = diff --git a/crates/load-gen/src/error.rs b/crates/load-gen/src/error.rs index 9747696..d13b814 100644 --- a/crates/load-gen/src/error.rs +++ b/crates/load-gen/src/error.rs @@ -11,6 +11,9 @@ pub enum LoadGenError { #[error("histogram error: {0}")] Histogram(#[from] hdrhistogram::RecordError), + #[error("histogram creation error: {0}")] + HistogramCreation(#[from] hdrhistogram::CreationError), + #[error("protocol error: {0}")] Protocol(String), diff --git a/crates/load-gen/src/histogram.rs b/crates/load-gen/src/histogram.rs index 99af32d..505bb2e 100644 --- a/crates/load-gen/src/histogram.rs +++ b/crates/load-gen/src/histogram.rs @@ -47,12 +47,6 @@ impl LatencyRecorder { } } -impl Default for LatencyRecorder { - fn default() -> Self { - Self::new().expect("default histogram creation cannot fail") - } -} - impl std::fmt::Debug for LatencyRecorder { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("LatencyRecorder") diff --git a/crates/load-gen/src/plan.rs b/crates/load-gen/src/plan.rs index 0a38f39..735c897 100644 --- a/crates/load-gen/src/plan.rs +++ b/crates/load-gen/src/plan.rs @@ -2,6 +2,8 @@ use crate::LoadGenError; pub const MAX_CONCURRENCY: usize = 256; pub const MAX_PIPELINE_DEPTH: usize = 64; +pub const MAX_DURATION_SECS: u64 = 3_600; +pub const MAX_TARGET_RPS: u64 = 10_000_000; #[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] pub enum ConnectionMode { @@ -40,17 +42,24 @@ impl BenchmarkPlan { if self.target_addr.trim().is_empty() { return Err(LoadGenError::InvalidPlan("target address is empty".into())); } - if !self.path.starts_with('/') { - return Err(LoadGenError::InvalidPlan("path must start with '/'".into())); + if !valid_target_addr(&self.target_addr) { + return Err(LoadGenError::InvalidPlan( + "target address contains invalid characters".into(), + )); + } + if !valid_request_path(&self.path) { + return Err(LoadGenError::InvalidPlan( + "path must start with '/' and contain no whitespace".into(), + )); } if !(1..=MAX_CONCURRENCY).contains(&self.concurrency) { return Err(LoadGenError::InvalidPlan( "concurrency must be between 1 and 256".into(), )); } - if self.duration_secs == 0 { + if !(1..=MAX_DURATION_SECS).contains(&self.duration_secs) { return Err(LoadGenError::InvalidPlan( - "duration must be positive".into(), + "duration must be between 1 and 3600 seconds".into(), )); } if !(1..=MAX_PIPELINE_DEPTH).contains(&self.pipeline_depth) { @@ -63,18 +72,39 @@ impl BenchmarkPlan { "churn mode requires a pipeline depth of 1".into(), )); } - if self.target_rps == Some(0) { + if self + .target_rps + .is_some_and(|rps| !(1..=MAX_TARGET_RPS).contains(&rps)) + { return Err(LoadGenError::InvalidPlan( - "target RPS must be positive".into(), + "target RPS must be between 1 and 10000000".into(), )); } Ok(()) } } +fn valid_target_addr(value: &str) -> bool { + value.len() <= 255 + && !value + .bytes() + .any(|byte| byte.is_ascii_control() || byte.is_ascii_whitespace()) +} + +fn valid_request_path(value: &str) -> bool { + value.starts_with('/') + && value.len() <= 2048 + && !value + .bytes() + .any(|byte| byte.is_ascii_control() || byte.is_ascii_whitespace()) +} + #[cfg(test)] mod tests { - use super::{BenchmarkPlan, ConnectionMode, MAX_CONCURRENCY, MAX_PIPELINE_DEPTH}; + use super::{ + BenchmarkPlan, ConnectionMode, MAX_CONCURRENCY, MAX_DURATION_SECS, MAX_PIPELINE_DEPTH, + MAX_TARGET_RPS, + }; #[test] fn validate_rejects_invalid_measurement_bounds() { @@ -112,6 +142,20 @@ mod tests { assert!(pipeline.validate().is_err()); } + #[test] + fn validate_rejects_unbounded_duration_and_rate() { + let long = BenchmarkPlan { + duration_secs: MAX_DURATION_SECS + 1, + ..BenchmarkPlan::default() + }; + let fast = BenchmarkPlan { + target_rps: Some(MAX_TARGET_RPS + 1), + ..BenchmarkPlan::default() + }; + assert!(long.validate().is_err()); + assert!(fast.validate().is_err()); + } + #[test] fn validate_rejects_pipelined_connection_churn() { let plan = BenchmarkPlan { diff --git a/crates/load-gen/src/response.rs b/crates/load-gen/src/response.rs index 8191507..4be9ed4 100644 --- a/crates/load-gen/src/response.rs +++ b/crates/load-gen/src/response.rs @@ -1,6 +1,7 @@ use crate::LoadGenError; pub(crate) const MAX_RESPONSE_BYTES: usize = 65_536; +pub(crate) const MAX_RESPONSE_HEADERS: usize = 64; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) struct HttpResponse { @@ -10,7 +11,7 @@ pub(crate) struct HttpResponse { } pub(crate) fn parse_response(bytes: &[u8]) -> Result, LoadGenError> { - let mut headers = [httparse::EMPTY_HEADER; 16]; + let mut headers = [httparse::EMPTY_HEADER; MAX_RESPONSE_HEADERS]; let (header_len, status) = parse_headers(bytes, &mut headers)?; let Some((header_len, status)) = header_len.zip(status) else { return Ok(None); diff --git a/crates/load-gen/tests/load_gen_test.rs b/crates/load-gen/tests/load_gen_test.rs index ec0a736..71b8f3d 100644 --- a/crates/load-gen/tests/load_gen_test.rs +++ b/crates/load-gen/tests/load_gen_test.rs @@ -3,7 +3,7 @@ use load_gen::stats::WorkerStats; #[test] fn test_latency_recorder_quantiles() { - let mut recorder = LatencyRecorder::default(); + let mut recorder = LatencyRecorder::new().expect("create latency histogram"); recorder.record_micros(100); recorder.record_micros(200); recorder.record_micros(300); @@ -18,7 +18,7 @@ fn test_latency_recorder_quantiles() { #[test] fn test_latency_recorder_coordinated_omission() { - let mut recorder = LatencyRecorder::default(); + let mut recorder = LatencyRecorder::new().expect("create latency histogram"); recorder.record_micros_corrected(10_000, 1_000); let dist = recorder.export_distribution(); diff --git a/crates/orchestrator/src/build_options.rs b/crates/orchestrator/src/build_options.rs index 0cf6e38..1cb810a 100644 --- a/crates/orchestrator/src/build_options.rs +++ b/crates/orchestrator/src/build_options.rs @@ -1,8 +1,10 @@ use crate::error::OrchestratorError; +use std::collections::HashMap; pub(crate) fn docker_build_args( dockerfile: &str, image: &str, + build_args: &HashMap, ) -> Result, OrchestratorError> { let mut args = vec![ "build".into(), @@ -13,6 +15,11 @@ pub(crate) fn docker_build_args( ]; append_enabled_option(&mut args, "BENCHMARK_IMAGE_PULL", "--pull")?; append_enabled_option(&mut args, "BENCHMARK_NO_CACHE", "--no-cache")?; + let mut build_args: Vec<_> = build_args.iter().collect(); + build_args.sort_unstable_by_key(|(key, _)| *key); + for (key, value) in build_args { + args.extend(["--build-arg".into(), format!("{key}={value}")]); + } args.push(".".into()); Ok(args) } @@ -41,7 +48,8 @@ fn parse_build_option(variable: &str, value: Option<&str>) -> Result docker.dockerfile_musl.as_deref(), } .ok_or_else(|| OrchestratorError::Process("framework lacks requested libc image".into()))?; + let dockerfile = crate::container_paths::secure_dockerfile(framework_dir, dockerfile)?; + let dockerfile = dockerfile + .to_str() + .ok_or_else(|| OrchestratorError::Process("dockerfile path must be valid UTF-8".into()))?; let cpuset_cpus = container_config::optional_config("BENCHMARK_TARGET_CPUS", container_config::valid_cpuset)?; let cgroup_parent = container_config::optional_config( @@ -45,37 +49,25 @@ pub fn start_container( .duration_since(UNIX_EPOCH) .map_err(|error| OrchestratorError::Process(error.to_string()))? .as_millis(); - let base = sanitize_name(&manifest.framework.name); + let base = crate::container_paths::sanitize_name(&manifest.framework.name); let image = format!("framework-benchmark/{base}:{suffix}"); let name = format!("benchmark-{base}-{suffix}-{stamp}"); - let build_args = crate::build_options::docker_build_args(dockerfile, &image)?; - run_docker(framework_dir, &build_args)?; + let build_args = + crate::build_options::docker_build_args(dockerfile, &image, &docker.build_args)?; + run_docker_stream(framework_dir, &build_args)?; let image_id = image_id(framework_dir, &image)?; - let mut args = vec![ - "run".into(), - "--detach".into(), - "--name".into(), - name.clone(), - "--label".into(), - "framework-benchmark.managed=true".into(), - "--network".into(), - "host".into(), - "--stop-signal".into(), - shutdown_signal.into(), - "--env".into(), - format!("BENCHMARK_WORKERS={}", benchmark_workers()), - ]; - args.extend(container_config::isolation_args( - cpuset_cpus.as_deref(), - cgroup_parent.as_deref(), - )); - if small_host() { - args.extend(["--cpus", "1", "--memory", "2g", "--pids-limit", "128"].map(String::from)); - } - if let Some(database_url) = database_url { - args.extend(["--env".into(), format!("DATABASE_URL={database_url}")]); - } - args.push(image); + let workers = crate::host_profile::target_workers(cpuset_cpus.as_deref()); + let spec = crate::container_args::RunSpec { + name: &name, + shutdown_signal, + workers, + cpuset_cpus: cpuset_cpus.as_deref(), + cgroup_parent: cgroup_parent.as_deref(), + database_url, + environment: &manifest.runtime.environment, + image: &image, + }; + let args = crate::container_args::run_args(&spec); let id = run_docker(framework_dir, &args)?.trim().to_owned(); let pid = match container_pid(framework_dir, &id) { Ok(pid) => pid, @@ -114,15 +106,11 @@ fn image_id(framework_dir: &Path, image: &str) -> Result Result<(), OrchestratorError> { - let stopped = run_docker(framework_dir, &container_config::stop_args(name)); - let removed = run_docker(framework_dir, &["rm", name]); - if removed.is_ok() { - return Ok(()); - } - if run_docker(framework_dir, &["rm", "--force", name]).is_ok() { + let _ = run_docker(framework_dir, &container_config::stop_args(name)); + if run_docker(framework_dir, &["rm", name]).is_ok() { return Ok(()); } - Err(stopped.and(removed).unwrap_err()) + run_docker(framework_dir, &["rm", "--force", name]).map(|_| ()) } fn container_pid(framework_dir: &Path, id: &str) -> Result { run_docker(framework_dir, &["inspect", "--format={{.State.Pid}}", id])? @@ -143,21 +131,14 @@ fn cgroup_dir_for_pid(pid: u32) -> Result { } fn cgroup_dir_from_content(content: &str) -> Option { let cgroup = content.lines().find_map(|line| line.strip_prefix("0::"))?; - Some(Path::new("/sys/fs/cgroup").join(cgroup.trim_start_matches('/'))) -} -fn benchmark_workers() -> usize { - env::var("BENCHMARK_WORKERS") - .ok() - .and_then(|value| value.parse().ok()) - .filter(|value: &usize| *value > 0) - .unwrap_or_else(|| std::thread::available_parallelism().map_or(1, usize::from)) -} -fn small_host() -> bool { - env::var("BENCHMARK_SMALL_HOST").is_ok_and(|value| value == "1") + let relative = cgroup.trim_start_matches('/'); + if relative.is_empty() { + return None; + } + Some(Path::new("/sys/fs/cgroup").join(relative)) } fn run_docker>(cwd: &Path, args: &[S]) -> Result { - let runtime = env::var("CONTAINER_RUNTIME").unwrap_or_else(|_| "docker".into()); - let output = Command::new(runtime) + let output = Command::new(container_runtime()) .args(args.iter().map(AsRef::as_ref)) .current_dir(cwd) .output()?; @@ -168,16 +149,22 @@ fn run_docker>(cwd: &Path, args: &[S]) -> Result String { - name.chars() - .map(|char| { - if char.is_ascii_alphanumeric() { - char - } else { - '-' - } - }) - .collect() + +fn run_docker_stream>(cwd: &Path, args: &[S]) -> Result<(), OrchestratorError> { + let status = Command::new(container_runtime()) + .args(args.iter().map(AsRef::as_ref)) + .current_dir(cwd) + .status()?; + if status.success() { + return Ok(()); + } + Err(OrchestratorError::Process(format!( + "container command failed with {status}" + ))) +} + +fn container_runtime() -> String { + env::var("CONTAINER_RUNTIME").unwrap_or_else(|_| "docker".into()) } #[cfg(test)] mod tests { @@ -192,7 +179,7 @@ mod tests { } #[test] - fn names_replace_container_unsafe_characters() { - assert_eq!(sanitize_name("rust/axum"), "rust-axum"); + fn cgroup_root_is_not_accepted_as_a_target_scope() { + assert_eq!(cgroup_dir_from_content("0::/\n"), None); } } diff --git a/crates/orchestrator/src/container_args.rs b/crates/orchestrator/src/container_args.rs new file mode 100644 index 0000000..faba54b --- /dev/null +++ b/crates/orchestrator/src/container_args.rs @@ -0,0 +1,121 @@ +use crate::host_profile; +use std::collections::HashMap; +use std::env; + +pub(crate) struct RunSpec<'a> { + pub name: &'a str, + pub shutdown_signal: &'a str, + pub workers: usize, + pub cpuset_cpus: Option<&'a str>, + pub cgroup_parent: Option<&'a str>, + pub database_url: Option<&'a str>, + pub environment: &'a HashMap, + pub image: &'a str, +} + +pub(crate) fn run_args(spec: &RunSpec<'_>) -> Vec { + let mut args = vec![ + "run".into(), + "--detach".into(), + "--name".into(), + spec.name.into(), + "--label".into(), + "framework-benchmark.managed=true".into(), + "--network".into(), + "host".into(), + "--init".into(), + "--stop-signal".into(), + spec.shutdown_signal.into(), + "--env".into(), + format!("BENCHMARK_WORKERS={}", spec.workers), + ]; + if !logs_enabled() { + args.extend(["--log-driver".into(), "none".into()]); + } + if hardening_enabled() { + args.extend([ + "--cap-drop".into(), + "ALL".into(), + "--security-opt".into(), + "no-new-privileges".into(), + ]); + } + args.extend(limit_args()); + if let Some(cpus) = spec.cpuset_cpus { + args.extend(["--cpuset-cpus".into(), cpus.into()]); + } + if let Some(parent) = spec.cgroup_parent { + args.extend(["--cgroup-parent".into(), parent.into()]); + } + let mut environment: Vec<_> = spec.environment.iter().collect(); + environment.sort_unstable_by_key(|(key, _)| *key); + for (key, value) in environment { + args.extend(["--env".into(), format!("{key}={value}")]); + } + if let Some(url) = spec.database_url { + args.extend(["--env".into(), format!("DATABASE_URL={url}")]); + } + args.push(spec.image.into()); + args +} + +fn limit_args() -> Vec { + let small = host_profile::small_host(); + let mut args = Vec::new(); + let pids = env::var("BENCHMARK_PIDS_LIMIT") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|value| *value > 0) + .unwrap_or(if small { 128 } else { 512 }); + args.extend(["--pids-limit".into(), pids.to_string()]); + if small { + args.extend(["--cpus".into(), "1".into()]); + } + let memory = env::var("BENCHMARK_TARGET_MEMORY") + .ok() + .filter(|value| !value.trim().is_empty()) + .or_else(|| small.then(|| "2g".into())); + if let Some(memory) = memory { + args.extend(["--memory".into(), memory.clone()]); + args.extend(["--memory-swap".into(), memory]); + } + args +} + +fn logs_enabled() -> bool { + host_profile::env_flag("BENCHMARK_CONTAINER_LOGS") +} + +fn hardening_enabled() -> bool { + !host_profile::env_flag("BENCHMARK_DISABLE_HARDENING") +} + +#[cfg(test)] +mod tests { + use super::{RunSpec, run_args}; + use std::collections::HashMap; + + #[test] + fn run_arguments_keep_isolation_and_environment_boundaries() { + let environment = HashMap::from([ + ("Z_VAR".into(), "last".into()), + ("A_VAR".into(), "first".into()), + ]); + let spec = RunSpec { + name: "benchmark-test", + shutdown_signal: "SIGTERM", + workers: 2, + cpuset_cpus: Some("0-1"), + cgroup_parent: Some("benchmark.slice"), + database_url: Some("postgres://example/db"), + environment: &environment, + image: "framework-benchmark/test:glibc", + }; + let args = run_args(&spec).join(" "); + assert!(args.contains("--cpuset-cpus 0-1")); + assert!(args.contains("--cgroup-parent benchmark.slice")); + assert!(args.contains("--env A_VAR=first --env Z_VAR=last")); + assert!(args.contains("DATABASE_URL=postgres://example/db")); + assert!(args.ends_with("framework-benchmark/test:glibc")); + } +} diff --git a/crates/orchestrator/src/container_config.rs b/crates/orchestrator/src/container_config.rs index 95c92da..b7b06df 100644 --- a/crates/orchestrator/src/container_config.rs +++ b/crates/orchestrator/src/container_config.rs @@ -53,20 +53,6 @@ pub(super) fn shutdown_signal(value: Option<&str>) -> Result<&str, OrchestratorE ))) } -pub(super) fn isolation_args( - cpuset_cpus: Option<&str>, - cgroup_parent: Option<&str>, -) -> Vec { - let mut args = Vec::with_capacity(4); - if let Some(cpus) = cpuset_cpus { - args.extend(["--cpuset-cpus".into(), cpus.into()]); - } - if let Some(parent) = cgroup_parent { - args.extend(["--cgroup-parent".into(), parent.into()]); - } - args -} - pub(super) fn stop_args(name: &str) -> [&str; 4] { ["stop", "--time", "5", name] } @@ -133,11 +119,7 @@ mod tests { } #[test] - fn docker_arguments_include_explicit_isolation_and_timeout() { - assert_eq!( - isolation_args(Some("2-7"), Some("benchmark.slice")), - ["--cpuset-cpus", "2-7", "--cgroup-parent", "benchmark.slice"] - ); + fn docker_stop_arguments_include_a_graceful_timeout() { assert_eq!(stop_args("target"), ["stop", "--time", "5", "target"]); } } diff --git a/crates/orchestrator/src/container_guard.rs b/crates/orchestrator/src/container_guard.rs new file mode 100644 index 0000000..ccb26c5 --- /dev/null +++ b/crates/orchestrator/src/container_guard.rs @@ -0,0 +1,33 @@ +use crate::container::stop_container; +use crate::error::OrchestratorError; +use std::path::Path; + +pub(crate) struct ContainerGuard<'a> { + framework_dir: &'a Path, + name: String, + armed: bool, +} + +impl<'a> ContainerGuard<'a> { + pub(crate) fn new(framework_dir: &'a Path, name: &str) -> Self { + Self { + framework_dir, + name: name.into(), + armed: true, + } + } + + pub(crate) fn stop(&mut self) -> Result<(), OrchestratorError> { + self.armed = false; + stop_container(self.framework_dir, &self.name) + } +} + +impl Drop for ContainerGuard<'_> { + fn drop(&mut self) { + if !self.armed { + return; + } + let _ = stop_container(self.framework_dir, &self.name); + } +} diff --git a/crates/orchestrator/src/container_paths.rs b/crates/orchestrator/src/container_paths.rs new file mode 100644 index 0000000..bc5be0e --- /dev/null +++ b/crates/orchestrator/src/container_paths.rs @@ -0,0 +1,69 @@ +use crate::error::OrchestratorError; +use std::fs; +use std::path::{Path, PathBuf}; + +pub(crate) fn sanitize_name(name: &str) -> String { + name.chars() + .map(|character| { + if character.is_ascii_lowercase() || character.is_ascii_digit() { + return character; + } + if character.is_ascii_uppercase() { + return character.to_ascii_lowercase(); + } + '-' + }) + .collect() +} + +pub(crate) fn secure_dockerfile(root: &Path, relative: &str) -> Result { + let root = fs::canonicalize(root)?; + let candidate = root.join(relative); + let metadata = fs::symlink_metadata(&candidate)?; + if metadata.file_type().is_symlink() { + return Err(OrchestratorError::Process( + "dockerfile must not be a symbolic link".into(), + )); + } + let path = fs::canonicalize(&candidate)?; + if !path.starts_with(&root) || !path.is_file() { + return Err(OrchestratorError::Process( + "dockerfile must remain inside the framework directory".into(), + )); + } + Ok(path) +} + +#[cfg(all(test, unix))] +mod tests { + use super::{sanitize_name, secure_dockerfile}; + use std::fs; + use std::os::unix::fs::symlink; + + #[test] + fn dockerfile_cannot_escape_or_use_a_symlink() { + let root = std::env::temp_dir().join(format!("dockerfile-root-{}", std::process::id())); + let outside = std::env::temp_dir().join(format!("dockerfile-out-{}", std::process::id())); + fs::create_dir_all(&root).expect("create root"); + fs::create_dir_all(&outside).expect("create outside"); + fs::write(root.join("Dockerfile.glibc"), "FROM scratch\n").expect("write dockerfile"); + fs::write(outside.join("Dockerfile.glibc"), "FROM scratch\n").expect("write outside"); + symlink( + outside.join("Dockerfile.glibc"), + root.join("Dockerfile.musl"), + ) + .expect("create symlink"); + + assert!(secure_dockerfile(&root, "Dockerfile.glibc").is_ok()); + assert!(secure_dockerfile(&root, "Dockerfile.musl").is_err()); + assert!(secure_dockerfile(&root, "../Dockerfile.glibc").is_err()); + + let _ = fs::remove_dir_all(root); + let _ = fs::remove_dir_all(outside); + } + + #[test] + fn container_names_are_lowercase_and_safe() { + assert_eq!(sanitize_name("Rust/Axum"), "rust-axum"); + } +} diff --git a/crates/orchestrator/src/database.rs b/crates/orchestrator/src/database.rs index c5f5151..d2149cc 100644 --- a/crates/orchestrator/src/database.rs +++ b/crates/orchestrator/src/database.rs @@ -27,7 +27,10 @@ pub fn start_database(repo_root: &Path) -> Result, Orche if was_running { return Ok(Some(session)); } - compose(&session, &["up", "--detach", "--wait"])?; + if let Err(error) = compose(&session, &["up", "--detach", "--wait"]) { + let _ = compose(&session, &["down", "--remove-orphans"]); + return Err(error); + } Ok(Some(DatabaseSession { started: true, ..session diff --git a/crates/orchestrator/src/discovery.rs b/crates/orchestrator/src/discovery.rs index dbde080..dd208f9 100644 --- a/crates/orchestrator/src/discovery.rs +++ b/crates/orchestrator/src/discovery.rs @@ -3,11 +3,14 @@ use common::manifest::{FrameworkManifest, parse_manifest}; use std::fs; use std::path::{Path, PathBuf}; +const MAX_DISCOVERY_DEPTH: usize = 16; + pub fn discover_frameworks( root: &Path, ) -> Result, OrchestratorError> { let mut discovered = Vec::new(); - find_manifests_recursive(root, &mut discovered)?; + find_manifests_recursive(root, &mut discovered, 0)?; + discovered.sort_unstable_by(|left, right| left.0.cmp(&right.0)); Ok(discovered) } @@ -34,7 +37,13 @@ pub fn select_frameworks( fn find_manifests_recursive( dir: &Path, results: &mut Vec<(PathBuf, FrameworkManifest)>, + depth: usize, ) -> Result<(), OrchestratorError> { + if depth > MAX_DISCOVERY_DEPTH { + return Err(OrchestratorError::Process( + "framework discovery exceeded the maximum directory depth".into(), + )); + } if !dir.is_dir() { return Ok(()); } @@ -42,17 +51,25 @@ fn find_manifests_recursive( let entries = fs::read_dir(dir)?; for entry in entries { let entry = entry?; - let path = entry.path(); - - if path.is_dir() { + let file_type = entry.file_type()?; + if file_type.is_symlink() { + continue; + } + if file_type.is_dir() { + let path = entry.path(); let manifest_file = path.join("framework.toml"); - if manifest_file.is_file() { - let content = fs::read_to_string(&manifest_file)?; - let manifest = parse_manifest(&content)?; - results.push((path, manifest)); - continue; + if let Ok(metadata) = fs::symlink_metadata(&manifest_file) { + if metadata.file_type().is_symlink() { + continue; + } + if metadata.is_file() { + let content = fs::read_to_string(&manifest_file)?; + let manifest = parse_manifest(&content)?; + results.push((path, manifest)); + continue; + } } - find_manifests_recursive(&path, results)?; + find_manifests_recursive(&path, results, depth + 1)?; } } Ok(()) diff --git a/crates/orchestrator/src/host_profile.rs b/crates/orchestrator/src/host_profile.rs new file mode 100644 index 0000000..3709a12 --- /dev/null +++ b/crates/orchestrator/src/host_profile.rs @@ -0,0 +1,94 @@ +use std::env; + +pub const SMALL_HOST_ENV: &str = "BENCHMARK_SMALL_HOST"; + +pub fn small_host() -> bool { + env_flag(SMALL_HOST_ENV) +} + +pub fn env_flag(name: &str) -> bool { + env::var(name).is_ok_and(|value| value == "1") +} + +pub fn target_host() -> String { + env::var("BENCHMARK_TARGET_HOST") + .ok() + .filter(|value| valid_host(value)) + .unwrap_or_else(|| "127.0.0.1".into()) +} + +pub fn target_addr(host: &str, port: u16) -> String { + if host.contains(':') && !host.starts_with('[') { + return format!("[{host}]:{port}"); + } + format!("{host}:{port}") +} + +pub fn target_is_remote(host: &str) -> bool { + !matches!(host, "127.0.0.1" | "localhost" | "::1") +} + +fn valid_host(value: &str) -> bool { + !value.trim().is_empty() + && value.len() <= 253 + && !value + .bytes() + .any(|byte| byte.is_ascii_control() || byte.is_ascii_whitespace()) +} + +pub fn target_workers(cpuset: Option<&str>) -> usize { + if let Some(workers) = env::var("BENCHMARK_TARGET_WORKERS") + .ok() + .and_then(|value| value.parse().ok()) + .filter(|value: &usize| *value > 0) + { + return workers; + } + if small_host() { + return 1; + } + if let Some(count) = cpuset.and_then(cpu_count) { + return count.max(1); + } + std::thread::available_parallelism().map_or(1, usize::from) +} + +pub fn cpu_count(cpuset: &str) -> Option { + let mut total = 0_usize; + for part in cpuset.split(',') { + let (start, end) = part.split_once('-').unwrap_or((part, part)); + let start = start.parse::().ok()?; + let end = end.parse::().ok()?; + if start > end { + return None; + } + total = total.checked_add(end - start + 1)?; + } + Some(total) +} + +#[cfg(test)] +mod tests { + use super::{cpu_count, target_addr, target_is_remote}; + + #[test] + fn cpu_sets_are_counted_without_overflows() { + assert_eq!(cpu_count("0"), Some(1)); + assert_eq!(cpu_count("2-7"), Some(6)); + assert_eq!(cpu_count("0,2-4"), Some(4)); + assert_eq!(cpu_count("4-2"), None); + } + + #[test] + fn only_loopback_hosts_are_local() { + assert!(!target_is_remote("127.0.0.1")); + assert!(!target_is_remote("localhost")); + assert!(target_is_remote("10.0.0.12")); + } + + #[test] + fn ipv6_hosts_are_bracketed_for_socket_addresses() { + assert_eq!(target_addr("::1", 8080), "[::1]:8080"); + assert_eq!(target_addr("127.0.0.1", 8080), "127.0.0.1:8080"); + } +} diff --git a/crates/orchestrator/src/lib.rs b/crates/orchestrator/src/lib.rs index e657c26..e3a6970 100644 --- a/crates/orchestrator/src/lib.rs +++ b/crates/orchestrator/src/lib.rs @@ -1,11 +1,17 @@ mod build_options; +mod container_args; +mod container_guard; +mod container_paths; pub mod discovery; pub mod error; +pub mod host_profile; pub mod lifecycle; pub mod machine; pub mod poller; mod provenance; pub mod runner; +mod session_results; +mod suite_timing; mod workload; pub mod ws_bench; diff --git a/crates/orchestrator/src/lifecycle.rs b/crates/orchestrator/src/lifecycle.rs index f0ddd36..f7e9d17 100644 --- a/crates/orchestrator/src/lifecycle.rs +++ b/crates/orchestrator/src/lifecycle.rs @@ -3,20 +3,7 @@ use common::manifest::FrameworkManifest; use common::result::{LibcVariant, RunProvenance, RunResult}; use std::path::Path; -#[derive(Debug, Clone, Copy)] -pub struct SuiteTiming { - pub warmup_secs: u64, - pub cooldown_secs: u64, -} - -impl Default for SuiteTiming { - fn default() -> Self { - Self { - warmup_secs: 15, - cooldown_secs: 10, - } - } -} +pub use crate::suite_timing::SuiteTiming; #[derive(Debug, Clone, Copy)] pub struct SuiteExecutionConfig<'a> { @@ -46,7 +33,7 @@ pub async fn run_framework_test( libc, target_pid: None, cgroup_dir: None, - timing: SuiteTiming::default(), + timing: SuiteTiming::for_environment(), extended_profiles: false, }; let metrics = @@ -70,7 +57,7 @@ pub async fn run_framework_suite( libc, target_pid, cgroup_dir, - timing: SuiteTiming::default(), + timing: SuiteTiming::for_environment(), extended_profiles: true, }; run_framework_suite_with_timing(manifest, config).await @@ -99,6 +86,7 @@ pub async fn run_framework_suite_with_timing( config.duration_secs, config.target_pid, config.cgroup_dir, + config.timing, ) .await?, ); diff --git a/crates/orchestrator/src/main.rs b/crates/orchestrator/src/main.rs index 6a17a66..1c9b63e 100644 --- a/crates/orchestrator/src/main.rs +++ b/crates/orchestrator/src/main.rs @@ -13,7 +13,10 @@ async fn main() -> Result<(), OrchestratorError> { let frameworks_dir = args.get(1).map_or("frameworks", String::as_str); let output_dir = args.get(2).map_or("results", String::as_str); - println!("==> Farben Framework Benchmark Orchestrator v0.1.0"); + println!( + "==> Farben Framework Benchmark Orchestrator v{}", + env!("CARGO_PKG_VERSION") + ); println!("==> Scanning framework registry at: {frameworks_dir}"); let filter = env::var("BENCHMARK_FRAMEWORK_FILTER").unwrap_or_else(|_| "all".into()); @@ -34,7 +37,10 @@ async fn main() -> Result<(), OrchestratorError> { || capabilities.fortunes || capabilities.updates }); - let database = if has_database_workloads { + let manage_container = BenchmarkSessionConfig::default().manage_container; + let manage_database = + env::var("BENCHMARK_DATABASE_MANAGED").map_or(manage_container, |value| value != "0"); + let database = if has_database_workloads && manage_database { start_database(Path::new("."))? } else { None @@ -45,7 +51,6 @@ async fn main() -> Result<(), OrchestratorError> { database_url: env::var("DATABASE_URL") .ok() .or_else(|| database.as_ref().map(|_| DEFAULT_DATABASE_URL.into())), - manage_container: env::var("BENCHMARK_MANAGE_CONTAINER").map_or(true, |value| value != "0"), ..BenchmarkSessionConfig::default() }; @@ -88,6 +93,7 @@ async fn main() -> Result<(), OrchestratorError> { ))); } if !results.is_empty() { + reporter::write_leaderboard_artifacts(Path::new(output_dir), &results)?; update_readme_leaderboard(Path::new("README.md"), &results)?; } Ok(()) @@ -125,8 +131,14 @@ fn require_dedicated_host_for_matrix(filter: &str) -> Result<(), OrchestratorErr if env::var("BENCHMARK_DEDICATED_HOST").is_ok_and(|value| value == "1") { return Ok(()); } + if orchestrator::host_profile::small_host() { + println!( + "==> Small-host profile enabled; use BENCHMARK_FRAMEWORK_FILTER for a bounded matrix." + ); + return Ok(()); + } Err(OrchestratorError::Process( - "full matrix requires BENCHMARK_DEDICATED_HOST=1".into(), + "full matrix requires BENCHMARK_DEDICATED_HOST=1 or BENCHMARK_SMALL_HOST=1".into(), )) } diff --git a/crates/orchestrator/src/poller.rs b/crates/orchestrator/src/poller.rs index cbcaceb..45a5017 100644 --- a/crates/orchestrator/src/poller.rs +++ b/crates/orchestrator/src/poller.rs @@ -3,6 +3,8 @@ use std::time::{Duration, Instant}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpStream; +const MAX_READY_RESPONSE_BYTES: usize = 2048; + pub async fn wait_for_ready( target_addr: &str, healthcheck_path: &str, @@ -36,7 +38,7 @@ async fn probe(target_addr: &str, request: &str, deadline: Instant) -> bool { if stream.write_all(request.as_bytes()).await.is_err() { return false; } - let mut response = [0_u8; 512]; + let mut response = [0_u8; MAX_READY_RESPONSE_BYTES]; let mut used = 0; while used < response.len() { let Ok(Ok(read)) = @@ -52,8 +54,43 @@ async fn probe(target_addr: &str, request: &str, deadline: Instant) -> bool { .windows(4) .any(|bytes| bytes == b"\r\n\r\n") { - return response[..used].starts_with(b"HTTP/1.1 200 "); + return is_success_response(&response[..used]); } } false } + +fn is_success_response(response: &[u8]) -> bool { + let Some(line_end) = response.iter().position(|byte| *byte == b'\n') else { + return false; + }; + let mut parts = response[..line_end].split(|byte| *byte == b' '); + let Some(version) = parts.next() else { + return false; + }; + if !version.starts_with(b"HTTP/") { + return false; + } + let Some(status) = parts.next() else { + return false; + }; + let Ok(status) = std::str::from_utf8(status) else { + return false; + }; + status + .parse::() + .is_ok_and(|code| (200..300).contains(&code)) +} + +#[cfg(test)] +mod tests { + use super::is_success_response; + + #[test] + fn any_2xx_status_is_ready() { + assert!(is_success_response(b"HTTP/1.1 200 OK\r\n\r\n")); + assert!(is_success_response(b"HTTP/1.0 204 No Content\r\n\r\n")); + assert!(!is_success_response(b"HTTP/1.1 503 Busy\r\n\r\n")); + assert!(!is_success_response(b"garbage\r\n\r\n")); + } +} diff --git a/crates/orchestrator/src/provenance.rs b/crates/orchestrator/src/provenance.rs index 7490a6f..7511262 100644 --- a/crates/orchestrator/src/provenance.rs +++ b/crates/orchestrator/src/provenance.rs @@ -17,7 +17,13 @@ pub fn framework_input_sha256(root: &Path) -> Result digest.update([0]); digest.update(fs::read(path)?); } - Ok(format!("{:x}", digest.finalize())) + let digest = digest.finalize(); + let mut hex = String::with_capacity(digest.len() * 2); + for byte in digest { + use std::fmt::Write; + let _ = write!(hex, "{byte:02x}"); + } + Ok(hex) } fn collect_files( diff --git a/crates/orchestrator/src/runner.rs b/crates/orchestrator/src/runner.rs index 5977af0..218d94e 100644 --- a/crates/orchestrator/src/runner.rs +++ b/crates/orchestrator/src/runner.rs @@ -1,18 +1,13 @@ +use crate::container_guard::ContainerGuard; use crate::error::OrchestratorError; -use crate::lifecycle::run_framework_suite; +use crate::lifecycle::{SuiteExecutionConfig, SuiteTiming, run_framework_suite_with_timing}; use crate::poller::wait_for_ready; use crate::provenance::framework_input_sha256; -use crate::{start_container, stop_container}; +use crate::session_results::persist_session_results; +use crate::start_container; use common::manifest::FrameworkManifest; use common::result::{LibcVariant, RunProvenance, RunResult}; -use reporter::archive::{ - timestamped_result_path, timestamped_result_relative_path, write_result_json, -}; -use reporter::chart::generate_throughput_svg; -use reporter::index::update_historical_index; -use reporter::logic::aggregate_leaderboard; -use reporter::markdown::{render_markdown_table, update_readme_leaderboard}; -use std::fs; +use reporter::markdown::update_readme_leaderboard; use std::path::{Path, PathBuf}; #[derive(Debug, Clone)] @@ -27,22 +22,31 @@ pub struct BenchmarkSessionConfig { pub cgroup_dir: Option, pub database_url: Option, pub manage_container: bool, + pub timing: SuiteTiming, } impl Default for BenchmarkSessionConfig { fn default() -> Self { - let small_host = std::env::var("BENCHMARK_SMALL_HOST").is_ok_and(|value| value == "1"); + let small_host = crate::host_profile::small_host(); + let target_host = crate::host_profile::target_host(); + let remote = crate::host_profile::target_is_remote(&target_host); + let manage_container = + std::env::var("BENCHMARK_MANAGE_CONTAINER").map_or(!remote, |value| value != "0"); + let default_duration = if small_host { 10 } else { 30 }; Self { - target_host: "127.0.0.1".into(), - concurrency: default_concurrency(small_host), - duration_secs: if small_host { 10 } else { 30 }, + target_host, + concurrency: configured_concurrency(small_host), + duration_secs: configured_duration(default_duration), libc: LibcVariant::Glibc, output_dir: "results".into(), readme_path: Some("README.md".into()), - target_pid: None, - cgroup_dir: None, + target_pid: std::env::var("BENCHMARK_TARGET_PID") + .ok() + .and_then(|value| value.parse().ok()), + cgroup_dir: std::env::var_os("BENCHMARK_TARGET_CGROUP").map(PathBuf::from), database_url: None, - manage_container: true, + manage_container, + timing: SuiteTiming::for_environment(), } } } @@ -55,6 +59,22 @@ fn default_concurrency(small_host: bool) -> u32 { u32::try_from(cpus.saturating_mul(4).clamp(4, 64)).unwrap_or(64) } +fn configured_concurrency(small_host: bool) -> u32 { + std::env::var("BENCHMARK_CONCURRENCY") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|value| (1..=256).contains(value)) + .unwrap_or_else(|| default_concurrency(small_host)) +} + +fn configured_duration(default: u32) -> u32 { + std::env::var("BENCHMARK_DURATION_SECS") + .ok() + .and_then(|value| value.parse::().ok()) + .filter(|value| (1..=3600).contains(value)) + .unwrap_or(default) +} + pub async fn execute_framework_session( framework_dir: &Path, manifest: &FrameworkManifest, @@ -72,8 +92,10 @@ pub async fn execute_framework_session( } else { None }; - let port = manifest.runtime.port; - let target_addr = format!("{}:{}", config.target_host, port); + let mut guard = target + .as_ref() + .map(|target| ContainerGuard::new(framework_dir, &target.name)); + let target_addr = crate::host_profile::target_addr(&config.target_host, manifest.runtime.port); let result = async { wait_for_ready( @@ -89,16 +111,17 @@ pub async fn execute_framework_session( .as_ref() .and_then(|target| target.cgroup_dir.as_deref()) .or(config.cgroup_dir.as_deref()); - let mut result = run_framework_suite( - manifest, - &target_addr, - config.concurrency, - config.duration_secs, - config.libc, + let suite = SuiteExecutionConfig { + target_addr: &target_addr, + concurrency: config.concurrency, + duration_secs: config.duration_secs, + libc: config.libc, target_pid, cgroup_dir, - ) - .await?; + timing: config.timing, + extended_profiles: true, + }; + let mut result = run_framework_suite_with_timing(manifest, suite).await?; result.provenance = RunProvenance { revision, image_id: target @@ -122,8 +145,8 @@ pub async fn execute_framework_session( Ok(result) } .await; - let cleanup = match &target { - Some(target) => stop_container(framework_dir, &target.name), + let cleanup = match guard.as_mut() { + Some(guard) => guard.stop(), None => Ok(()), }; match (result, cleanup) { @@ -133,10 +156,10 @@ pub async fn execute_framework_session( } fn current_revision(framework_dir: &Path) -> String { - if let Ok(revision) = std::env::var("BENCHMARK_REVISION") { - if !revision.trim().is_empty() { - return revision; - } + if let Ok(revision) = std::env::var("BENCHMARK_REVISION") + && !revision.trim().is_empty() + { + return revision; } std::process::Command::new("git") .args(["rev-parse", "HEAD"]) @@ -149,42 +172,3 @@ fn current_revision(framework_dir: &Path) -> String { .filter(|revision: &String| !revision.is_empty()) .unwrap_or_else(|| "unknown".into()) } - -fn persist_session_results(result: &RunResult, base_dir: &str) -> Result<(), OrchestratorError> { - let base = Path::new(base_dir); - let json_path = timestamped_result_path(base, result)?; - let relative_path = timestamped_result_relative_path(result)?; - let index_path = base.join("index.json"); - - write_result_json(&json_path, result)?; - update_historical_index( - &index_path, - result, - relative_path - .to_str() - .ok_or_else(|| OrchestratorError::Process("result path must be valid UTF-8".into()))?, - )?; - let charts_dir = base.join("charts"); - fs::create_dir_all(&charts_dir)?; - for category in categories(result) { - let rows = aggregate_leaderboard(std::slice::from_ref(result), &category); - let table = render_markdown_table(&rows, &category); - let title = format!("{category} Throughput Benchmark"); - let svg = generate_throughput_svg(&rows, &title); - fs::write(charts_dir.join(format!("{category}_throughput.svg")), svg)?; - fs::write(base.join(format!("leaderboard_{category}.md")), table)?; - } - - Ok(()) -} - -fn categories(result: &RunResult) -> Vec { - let mut categories: Vec = result - .benchmarks - .iter() - .map(|benchmark| benchmark.category.clone()) - .collect(); - categories.sort_unstable(); - categories.dedup(); - categories -} diff --git a/crates/orchestrator/src/session_results.rs b/crates/orchestrator/src/session_results.rs new file mode 100644 index 0000000..65dc5f7 --- /dev/null +++ b/crates/orchestrator/src/session_results.rs @@ -0,0 +1,27 @@ +use crate::error::OrchestratorError; +use common::result::RunResult; +use reporter::archive::{ + timestamped_result_path, timestamped_result_relative_path, write_result_json, +}; +use reporter::index::update_historical_index; +use std::path::Path; + +pub(crate) fn persist_session_results( + result: &RunResult, + base_dir: &str, +) -> Result<(), OrchestratorError> { + let base = Path::new(base_dir); + let json_path = timestamped_result_path(base, result)?; + let relative_path = timestamped_result_relative_path(result)?; + let index_path = base.join("index.json"); + + write_result_json(&json_path, result)?; + update_historical_index( + &index_path, + result, + relative_path + .to_str() + .ok_or_else(|| OrchestratorError::Process("result path must be valid UTF-8".into()))?, + )?; + Ok(()) +} diff --git a/crates/orchestrator/src/suite_timing.rs b/crates/orchestrator/src/suite_timing.rs new file mode 100644 index 0000000..825e661 --- /dev/null +++ b/crates/orchestrator/src/suite_timing.rs @@ -0,0 +1,44 @@ +use crate::host_profile; + +#[derive(Debug, Clone, Copy)] +pub struct SuiteTiming { + pub warmup_secs: u64, + pub cooldown_secs: u64, +} + +impl SuiteTiming { + pub const fn small() -> Self { + Self { + warmup_secs: 5, + cooldown_secs: 3, + } + } + + pub fn for_environment() -> Self { + if host_profile::small_host() { + return Self::small(); + } + Self::default() + } +} + +impl Default for SuiteTiming { + fn default() -> Self { + Self { + warmup_secs: 15, + cooldown_secs: 10, + } + } +} + +#[cfg(test)] +mod tests { + use super::SuiteTiming; + + #[test] + fn small_profile_reduces_warmup_and_cooldown() { + assert_eq!(SuiteTiming::small().warmup_secs, 5); + assert_eq!(SuiteTiming::small().cooldown_secs, 3); + assert!(SuiteTiming::small().warmup_secs < SuiteTiming::default().warmup_secs); + } +} diff --git a/crates/orchestrator/src/workload.rs b/crates/orchestrator/src/workload.rs index 8407a83..be459bf 100644 --- a/crates/orchestrator/src/workload.rs +++ b/crates/orchestrator/src/workload.rs @@ -1,6 +1,6 @@ use crate::lifecycle::{SuiteExecutionConfig, SuiteTiming}; use collector::BackgroundSampler; -use common::result::{TestMetrics, TrafficSettings}; +use common::result::{ResourceUsage, TestMetrics, TrafficSettings}; use load_gen::engine::execute_benchmark; use load_gen::{BenchmarkPlan, ConnectionMode}; use std::time::Duration; @@ -71,15 +71,23 @@ pub(crate) async fn execute_single_workload( config: SuiteExecutionConfig<'_>, workload: Workload<'_>, ) -> Result { - execute_benchmark(plan(config, workload, config.timing, true)).await?; - let sampler = BackgroundSampler::spawn_target( - config.target_pid, - config.cgroup_dir, - Duration::from_millis(50), - ); + if config.timing.warmup_secs > 0 { + execute_benchmark(plan(config, workload, config.timing, true)).await?; + } + let sampler = (config.target_pid.is_some() || config.cgroup_dir.is_some()).then(|| { + BackgroundSampler::spawn_target( + config.target_pid, + config.cgroup_dir, + Duration::from_millis(50), + ) + }); let result = execute_benchmark(plan(config, workload, config.timing, false)).await?; - let resources = sampler.stop().await; + let resources = match sampler { + Some(sampler) => sampler.stop().await, + None => ResourceUsage::default(), + }; tokio::time::sleep(Duration::from_secs(config.timing.cooldown_secs)).await; + validate_status_codes(workload.category, &result.stats)?; Ok(TestMetrics { category: workload.category.into(), concurrency: config.concurrency, @@ -94,6 +102,22 @@ pub(crate) async fn execute_single_workload( }) } +fn validate_status_codes( + category: &str, + stats: &load_gen::WorkerStats, +) -> Result<(), crate::error::OrchestratorError> { + let rejected = stats + .status_4xx + .saturating_add(stats.status_5xx) + .saturating_add(stats.errors); + if rejected == 0 { + return Ok(()); + } + Err(crate::error::OrchestratorError::Process(format!( + "workload {category} recorded {rejected} rejected or failed responses" + ))) +} + fn plan( config: SuiteExecutionConfig<'_>, workload: Workload<'_>, diff --git a/crates/orchestrator/src/ws_bench.rs b/crates/orchestrator/src/ws_bench.rs index ea64a61..40b1d45 100644 --- a/crates/orchestrator/src/ws_bench.rs +++ b/crates/orchestrator/src/ws_bench.rs @@ -1,5 +1,6 @@ +use crate::lifecycle::SuiteTiming; use collector::BackgroundSampler; -use common::result::{TestMetrics, TrafficSettings}; +use common::result::{ResourceUsage, TestMetrics, TrafficSettings}; use load_gen::autobahn::AutobahnSuiteRunner; use load_gen::histogram::LatencyRecorder; use load_gen::stats::WorkerStats; @@ -13,19 +14,90 @@ pub async fn execute_websocket_workload( duration_secs: u32, target_pid: Option, cgroup_dir: Option<&Path>, + timing: SuiteTiming, ) -> Result { - let sampler = - BackgroundSampler::spawn_target(target_pid, cgroup_dir, Duration::from_millis(50)); + let workers = concurrency.max(1); + if !(1..=256).contains(&workers) { + return Err(crate::error::OrchestratorError::Process( + "websocket concurrency must be between 1 and 256".into(), + )); + } + if duration_secs == 0 { + return Err(crate::error::OrchestratorError::Process( + "websocket duration must be positive".into(), + )); + } + if timing.warmup_secs > 0 { + let deadline = Instant::now() + Duration::from_secs(timing.warmup_secs); + let warmup_workers = workers.min(16); + run_round(target_addr, endpoint_path, warmup_workers, deadline).await?; + } + let sampler = (target_pid.is_some() || cgroup_dir.is_some()).then(|| { + BackgroundSampler::spawn_target(target_pid, cgroup_dir, Duration::from_millis(50)) + }); let started = Instant::now(); let deadline = started + Duration::from_secs(u64::from(duration_secs)); - let workers = concurrency.max(1); - let mut handles = Vec::with_capacity(workers as usize); + let result = run_round(target_addr, endpoint_path, workers, deadline).await; + let resources = match sampler { + Some(sampler) => sampler.stop().await, + None => ResourceUsage::default(), + }; + let (recorder, stats) = result?; + tokio::time::sleep(Duration::from_secs(timing.cooldown_secs)).await; + + let rejected = stats + .status_4xx + .saturating_add(stats.status_5xx) + .saturating_add(stats.errors); + if rejected > 0 { + return Err(crate::error::OrchestratorError::Process(format!( + "websocket workload recorded {rejected} rejected or failed responses" + ))); + } + + let elapsed_secs = started.elapsed().as_secs_f64().max(0.001); + let rps = stats.requests_completed as f64 / elapsed_secs; + let mb_per_sec = (stats.bytes_transferred as f64 / (1024.0 * 1024.0)) / elapsed_secs; + + Ok(TestMetrics { + category: "websocket".into(), + concurrency: workers, + duration_secs, + requests_total: stats.requests_completed, + requests_per_sec: rps, + transfer_mb_per_sec: mb_per_sec, + latency: recorder.export_distribution(), + resources, + status_codes: stats.to_status_code_counts(), + traffic: TrafficSettings { + pipeline_depth: 1, + connection_mode: "keep_alive".into(), + target_rps: None, + }, + }) +} + +fn new_recorder() -> Result { + LatencyRecorder::new().map_err(|error| crate::error::OrchestratorError::LoadGen(error.into())) +} + +async fn run_round( + target_addr: &str, + endpoint_path: &str, + workers: u32, + deadline: Instant, +) -> Result<(LatencyRecorder, WorkerStats), crate::error::OrchestratorError> { + let mut recorders = Vec::with_capacity(workers as usize); for _ in 0..workers { + recorders.push(new_recorder()?); + } + let mut handles = Vec::with_capacity(workers as usize); + for recorder in recorders { let addr = target_addr.to_string(); let path = endpoint_path.to_string(); handles.push(tokio::spawn(async move { - let mut recorder = LatencyRecorder::default(); + let mut recorder = recorder; let mut stats = WorkerStats::default(); let result = AutobahnSuiteRunner::run_echo_benchmark( &addr, @@ -39,7 +111,7 @@ pub async fn execute_websocket_workload( })); } - let mut recorder = LatencyRecorder::default(); + let mut recorder = new_recorder()?; let mut stats = WorkerStats::default(); let mut failure = None; for handle in handles { @@ -57,31 +129,10 @@ pub async fn execute_websocket_workload( } } } - - let resources = sampler.stop().await; if let Some(error) = failure { return Err(crate::error::OrchestratorError::Process(format!( "websocket workload failed: {error}" ))); } - let elapsed_secs = started.elapsed().as_secs_f64().max(0.001); - let rps = stats.requests_completed as f64 / elapsed_secs; - let mb_per_sec = (stats.bytes_transferred as f64 / (1024.0 * 1024.0)) / elapsed_secs; - - Ok(TestMetrics { - category: "websocket".into(), - concurrency, - duration_secs, - requests_total: stats.requests_completed, - requests_per_sec: rps, - transfer_mb_per_sec: mb_per_sec, - latency: recorder.export_distribution(), - resources, - status_codes: stats.to_status_code_counts(), - traffic: TrafficSettings { - pipeline_depth: 1, - connection_mode: "keep_alive".into(), - target_rps: None, - }, - }) + Ok((recorder, stats)) } diff --git a/crates/orchestrator/tests/discovery_test.rs b/crates/orchestrator/tests/discovery_test.rs index 07c1925..2b98c94 100644 --- a/crates/orchestrator/tests/discovery_test.rs +++ b/crates/orchestrator/tests/discovery_test.rs @@ -75,9 +75,29 @@ fn select_frameworks_rejects_an_unknown_name() { let _ = fs::remove_dir_all(temp_root); } +#[cfg(unix)] +#[test] +fn test_discovery_does_not_follow_symlinked_directories() { + use std::os::unix::fs::symlink; + + let root = std::env::temp_dir().join(format!("bench_disc_root_{}", std::process::id())); + let outside = std::env::temp_dir().join(format!("bench_disc_out_{}", std::process::id())); + let target = outside.join("linked"); + fs::create_dir_all(&target).expect("create outside target"); + fs::write(target.join("framework.toml"), SAMPLE_MANIFEST).expect("write manifest"); + fs::create_dir_all(&root).expect("create root"); + symlink(&outside, root.join("escape")).expect("create symlink"); + + let discovered = discover_frameworks(&root).expect("discovery failed"); + assert!(discovered.is_empty()); + + let _ = fs::remove_dir_all(root); + let _ = fs::remove_dir_all(outside); +} + #[test] fn test_registry_manifests_are_valid() { let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../frameworks"); let discovered = discover_frameworks(&root).expect("discover framework registry failed"); - assert_eq!(discovered.len(), 31); + assert!(discovered.len() >= 31); } diff --git a/crates/reporter/src/archive.rs b/crates/reporter/src/archive.rs index c4e6ac0..b33f4ba 100644 --- a/crates/reporter/src/archive.rs +++ b/crates/reporter/src/archive.rs @@ -1,4 +1,5 @@ use crate::error::ReporterError; +use crate::safety::safe_component; use crate::storage::write_new_file; use common::result::RunResult; use std::path::{Path, PathBuf}; @@ -11,20 +12,17 @@ pub fn timestamped_result_path( } pub fn timestamped_result_relative_path(result: &RunResult) -> Result { - if !valid_run_id(&result.run_id) { - return Err(ReporterError::Formatting( + let run_id = safe_component(&result.run_id).map_err(|_| { + ReporterError::Formatting( "run id must contain only ASCII letters, digits, hyphens, and underscores".into(), - )); - } + ) + })?; let (year, month) = year_month(result.timestamp_epoch_secs); Ok(PathBuf::from("data") .join(format!("{year:04}")) .join(format!("{month:02}")) - .join(format!( - "{}-{}.json", - result.timestamp_epoch_secs, result.run_id - ))) + .join(format!("{}-{run_id}.json", result.timestamp_epoch_secs))) } pub fn write_result_json(output_path: &Path, result: &RunResult) -> Result<(), ReporterError> { @@ -32,13 +30,6 @@ pub fn write_result_json(output_path: &Path, result: &RunResult) -> Result<(), R write_new_file(output_path, json.as_bytes()) } -fn valid_run_id(run_id: &str) -> bool { - !run_id.is_empty() - && run_id - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-' || byte == b'_') -} - fn year_month(timestamp_epoch_secs: u64) -> (u64, u64) { let days = timestamp_epoch_secs / 86_400 + 719_468; let era = days / 146_097; diff --git a/crates/reporter/src/artifacts.rs b/crates/reporter/src/artifacts.rs new file mode 100644 index 0000000..0899b12 --- /dev/null +++ b/crates/reporter/src/artifacts.rs @@ -0,0 +1,38 @@ +use crate::chart::generate_throughput_svg; +use crate::error::ReporterError; +use crate::logic::aggregate_leaderboard; +use crate::markdown::render_markdown_table; +use crate::safety::{safe_component, safe_file_name}; +use common::result::RunResult; +use std::fs; +use std::path::Path; + +pub fn write_leaderboard_artifacts( + base: &Path, + results: &[RunResult], +) -> Result<(), ReporterError> { + let charts_dir = base.join("charts"); + fs::create_dir_all(&charts_dir)?; + for category in categories(results) { + let rows = aggregate_leaderboard(results, &category); + let table = render_markdown_table(&rows, &category); + let title = format!("{category} Throughput Benchmark"); + let svg = generate_throughput_svg(&rows, &title); + let chart_name = safe_file_name(&category, "_throughput.svg")?; + let table_name = format!("leaderboard_{}.md", safe_component(&category)?); + fs::write(charts_dir.join(chart_name), svg)?; + fs::write(base.join(table_name), table)?; + } + Ok(()) +} + +fn categories(results: &[RunResult]) -> Vec { + let mut categories: Vec = results + .iter() + .flat_map(|result| result.benchmarks.iter()) + .map(|benchmark| benchmark.category.clone()) + .collect(); + categories.sort_unstable(); + categories.dedup(); + categories +} diff --git a/crates/reporter/src/chart.rs b/crates/reporter/src/chart.rs index b994f25..87180b2 100644 --- a/crates/reporter/src/chart.rs +++ b/crates/reporter/src/chart.rs @@ -107,10 +107,56 @@ fn xml_text(value: &str) -> String { #[cfg(test)] mod tests { - use super::xml_text; + use super::{generate_throughput_svg, xml_text}; + use crate::logic::LeaderboardRow; + + fn row(framework: &str) -> LeaderboardRow { + LeaderboardRow { + rank: 1, + framework: framework.into(), + framework_version: "1.0".into(), + libc: "glibc".into(), + concurrency: 64, + duration_secs: 10, + requests_total: 100, + requests_per_sec: 10.0, + transfer_mb_per_sec: 1.0, + min_latency_ms: 1.0, + p50_latency_ms: 1.0, + p75_latency_ms: 1.0, + p90_latency_ms: 1.0, + p99_latency_ms: 1.0, + p99_9_latency_ms: 1.0, + p99_99_latency_ms: 1.0, + max_latency_ms: 1.0, + latency_std_dev_ms: 0.1, + peak_rss_mb: 1.0, + avg_rss_mb: 1.0, + cpu_user_pct: 1.0, + cpu_kernel_pct: 1.0, + context_switches: 1, + rapl_joules: None, + http_2xx: 1, + http_4xx: 0, + http_5xx: 0, + errors: 0, + } + } #[test] fn xml_text_escapes_element_and_attribute_delimiters() { assert_eq!(xml_text("<&>\"'"), "<&>"'"); } + + #[test] + fn svg_output_escapes_untrusted_labels() { + let svg = generate_throughput_svg( + &[row("")], + "", + ); + + assert!(!svg.contains("