diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000..b77956b56 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,363 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Performance regression gate for Fesod. +# +# Runs the baseline benchmark suite (fesod-benchmark/baseline package) on release tags +# (e.g. 2.1.0-incubating) and on demand (workflow_dispatch), compares the results against +# the committed baseline (fesod-benchmark/baseline/jmh-baseline.json) and: +# - appends the comparison report to the job step summary +# - emits ::error/::warning annotations for regressions (visible on the run page) +# - fails the job when a regression exceeds the fail threshold with non-overlapping +# JMH error bars (high-confidence regressions only โ€” see benchmark.md for the +# rationale of the tiered gate) +# +# On tags with a regression, a follow-up job posts the report as a comment on the +# matching GitHub Release (or opens an issue) so maintainers cannot miss it. +# A passing tag run (or a dispatch with update_baseline) opens a PR that refreshes +# the committed baseline โ€” the baseline lifecycle follows releases. +# +# The baseline can only be regenerated on a GitHub runner (same hardware/JDK). +# Never commit a baseline generated on a local machine. + +name: Benchmark + +on: + push: + tags: + - '[0-9]+.*' + workflow_dispatch: + inputs: + update_baseline: + description: 'Run the suite and open a PR refreshing the committed baseline' + type: boolean + default: false + warn_threshold: + description: 'Regression percentage that triggers a warning' + type: number + default: 10 + fail_threshold: + description: 'Regression percentage that fails the job (beyond JMH error bars)' + type: number + default: 20 + +concurrency: + group: benchmark-${{ github.ref }} + cancel-in-progress: false + +permissions: + contents: read + +env: + BASELINE_FILE: fesod-benchmark/baseline/jmh-baseline.json + BASELINE_META: fesod-benchmark/baseline/baseline-meta.json + CURRENT_RESULT: fesod-benchmark/target/baseline-current.json + REPORT_FILE: fesod-benchmark/target/benchmark-report.md + +jobs: + benchmark: + name: Baseline comparison (JDK 17) + runs-on: ubuntu-24.04 + timeout-minutes: 45 + permissions: + contents: read + pull-requests: write + outputs: + bootstrap: ${{ steps.baseline.outputs.exists != 'true' }} + update_requested: ${{ github.event_name == 'workflow_dispatch' && inputs.update_baseline }} + is_tag: ${{ startsWith(github.ref, 'refs/tags/') }} + compare_outcome: ${{ steps.compare.outcome }} + tag: ${{ github.ref_name }} + jdk: ${{ steps.env.outputs.jdk }} + os: ${{ steps.env.outputs.os }} + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Set up JDK 17 + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '17' + + - name: Cache local Maven repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-benchmark-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven-benchmark- + + - name: Record environment + id: env + run: | + echo "jdk=$(java -version 2>&1 | head -n 1)" >> "$GITHUB_OUTPUT" + echo "os=$(uname -sr)" >> "$GITHUB_OUTPUT" + + - name: Check baseline presence + id: baseline + run: | + if [ -f "$BASELINE_FILE" ]; then + echo "exists=true" >> "$GITHUB_OUTPUT" + else + echo "exists=false" >> "$GITHUB_OUTPUT" + echo "::notice::No committed baseline found โ€” this run will bootstrap one." + fi + + - name: Build benchmark suite + run: ./mvnw -B -ntp -pl fesod-benchmark -am package -DskipTests + + - name: Run baseline suite + run: | + cd fesod-benchmark + java -cp target/benchmarks.jar org.apache.fesod.sheet.benchmark.baseline.BaselineRunner + + - name: Compare against baseline + id: compare + continue-on-error: true + run: | + java -cp fesod-benchmark/target/benchmarks.jar \ + org.apache.fesod.sheet.benchmark.baseline.BaselineComparator \ + --baseline "$BASELINE_FILE" \ + --baseline-meta "$BASELINE_META" \ + --current "$CURRENT_RESULT" \ + --warn "${{ inputs.warn_threshold || vars.BENCHMARK_WARN_PCT || '10' }}" \ + --fail "${{ inputs.fail_threshold || vars.BENCHMARK_FAIL_PCT || '20' }}" \ + --report "$REPORT_FILE" + + - name: Add report to job summary + if: always() + run: | + RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + { + echo "## ๐Ÿ“Š Fesod performance report" + echo + echo "Run \`${{ github.sha }}\` on \`ubuntu-24.04\` ยท ${{ steps.env.outputs.jdk }} ยท [full run](${RUN_URL})" + echo + if [ -f "$REPORT_FILE" ]; then + cat "$REPORT_FILE" + else + echo "_No comparison report was produced (suite run failed or was cancelled)." + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v4 + with: + name: benchmark-results + if-no-files-found: warn + path: | + ${{ env.CURRENT_RESULT }} + ${{ env.REPORT_FILE }} + + - name: Comment report on open PR (manual runs on a PR branch) + if: github.event_name == 'workflow_dispatch' && always() + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + BRANCH: ${{ github.ref_name }} + run: | + [ -f "$REPORT_FILE" ] || { echo "No report file โ€” skipping."; exit 0; } + PR_NUMBER=$(gh pr list --repo "$GITHUB_REPOSITORY" --head "$BRANCH" --state open \ + --json number -q '.[0].number' 2>/dev/null || true) + if [ -z "$PR_NUMBER" ]; then + echo "No open PR for branch '$BRANCH' โ€” report is in the job summary only." + exit 0 + fi + MARKER='' + RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + BODY_FILE=$(mktemp) + { + echo "$MARKER" + echo "### ๐Ÿ“Š Performance vs baseline โ€” run [${{ github.run_id }}](${RUN_URL})" + echo + cat "$REPORT_FILE" + } > "$BODY_FILE" + EXISTING=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \ + --paginate -q ".[] | select(.body | startswith(\"$MARKER\")) | .id" | head -n 1) + if [ -n "$EXISTING" ]; then + gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$EXISTING" \ + -F body=@"$BODY_FILE" > /dev/null + else + gh api -X POST "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \ + -F body=@"$BODY_FILE" > /dev/null + fi + echo "Report commented on PR #$PR_NUMBER." + + - name: Fail on regression + if: steps.compare.outcome == 'failure' && !(github.event_name == 'workflow_dispatch' && inputs.update_baseline) + run: | + echo "::error title=Performance regression::At least one benchmark regressed beyond the fail threshold. See the benchmark report in the job summary." + exit 1 + + notify-regression: + name: Notify maintainers (tag regression) + needs: benchmark + if: >- + always() && + !cancelled() && + needs.benchmark.outputs.compare_outcome == 'failure' && + needs.benchmark.outputs.is_tag == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: write + issues: write + steps: + - name: Download benchmark results + uses: actions/download-artifact@v4 + with: + name: benchmark-results + path: fesod-benchmark/target/ + + - name: Post report on release (or open an issue) + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ needs.benchmark.outputs.tag }} + run: | + if [ ! -f "$REPORT_FILE" ]; then + echo "No report file โ€” nothing to notify." + exit 0 + fi + BODY_FILE=$(mktemp) + { + echo "## ๐Ÿšจ Performance regression detected on tag \`${TAG}\`" + echo + echo "The [benchmark run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) found regressions beyond the fail threshold. The baseline was **not** advanced." + echo + cat "$REPORT_FILE" + } > "$BODY_FILE" + + # Best-effort notification ladder: release comment -> issue -> warning + # annotation. A missing release or a repository with issues disabled must + # never turn this job red โ€” the gate itself is already red in that case. + notified=1 + RELEASE_ID=$(gh api "repos/$GITHUB_REPOSITORY/releases/tags/$TAG" -q .id 2>/dev/null || true) + # a failed lookup can leak the error body into the variable โ€” keep numeric ids only + case "$RELEASE_ID" in + ''|*[!0-9]*) RELEASE_ID='' ;; + esac + if [ -n "$RELEASE_ID" ]; then + if gh api -X POST "repos/$GITHUB_REPOSITORY/releases/$RELEASE_ID/comments" \ + -F body=@"$BODY_FILE" > /dev/null 2>&1; then + echo "Commented on release $TAG." + notified=0 + else + echo "::warning title=Regression notification::Release comment failed for '$TAG' โ€” falling back to an issue." + fi + fi + if [ "$notified" -ne 0 ]; then + if gh issue create --repo "$GITHUB_REPOSITORY" \ + --title "๐Ÿšจ Performance regression on tag ${TAG}" \ + --body-file "$BODY_FILE" --label "performance" 2>/dev/null \ + || gh issue create --repo "$GITHUB_REPOSITORY" \ + --title "๐Ÿšจ Performance regression on tag ${TAG}" \ + --body-file "$BODY_FILE"; then + echo "Opened a tracking issue." + notified=0 + fi + fi + if [ "$notified" -ne 0 ]; then + echo "::warning title=Regression notification::Could not comment on release '$TAG' or open an issue (no release / issues disabled?). The report is in the job summary and run annotations." + fi + + update-baseline: + name: Refresh baseline (opens PR) + needs: benchmark + if: >- + !cancelled() && + needs.benchmark.result == 'success' && + (needs.benchmark.outputs.update_requested == 'true' || + needs.benchmark.outputs.bootstrap == 'true' || + (needs.benchmark.outputs.is_tag == 'true' && needs.benchmark.outputs.compare_outcome == 'success')) + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: write + pull-requests: write + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Download benchmark results + uses: actions/download-artifact@v4 + with: + name: benchmark-results + path: fesod-benchmark/target/ + + - name: Write baseline files + env: + JDK_INFO: ${{ needs.benchmark.outputs.jdk }} + OS_INFO: ${{ needs.benchmark.outputs.os }} + SOURCE_REF: ${{ needs.benchmark.outputs.is_tag == 'true' && needs.benchmark.outputs.tag || github.ref_name }} + run: | + mkdir -p fesod-benchmark/baseline + cp "$CURRENT_RESULT" "$BASELINE_FILE" + jq -n \ + --arg generatedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg gitSha "${{ github.sha }}" \ + --arg gitRef "$SOURCE_REF" \ + --arg jdkVersion "$JDK_INFO" \ + --arg osName "$OS_INFO" \ + --arg runnerLabel "ubuntu-24.04" \ + --argjson benchmarkCount "$(jq 'length' "$CURRENT_RESULT")" \ + '{generatedAt: $generatedAt, gitSha: $gitSha, gitRef: $gitRef, jdkVersion: $jdkVersion, + osName: $osName, runnerLabel: $runnerLabel, benchmarkCount: $benchmarkCount}' \ + > "$BASELINE_META" + cat "$BASELINE_META" + + - name: Open baseline refresh PR + env: + GH_TOKEN: ${{ github.token }} + BASE_BRANCH: ${{ needs.benchmark.outputs.is_tag == 'true' && 'main' || github.ref_name }} + REASON: ${{ needs.benchmark.outputs.is_tag == 'true' && format('release tag {0}', needs.benchmark.outputs.tag) || github.ref_name }} + run: | + BRANCH="benchmark/baseline-$(date -u +%Y%m%d%H%M%S)" + PR_BODY_FILE=$(mktemp) + { + echo "## Performance baseline refresh" + echo + echo "Regenerated from \`${{ github.sha }}\` (\`$REASON\`) on a \`ubuntu-24.04\` runner with JDK 17 (Temurin)" + echo "by the [Benchmark workflow](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})." + echo + if [ -f "$REPORT_FILE" ]; then + echo "### Change vs previous baseline" + echo + cat "$REPORT_FILE" + echo + fi + echo "### Files changed" + echo + echo '- `fesod-benchmark/baseline/jmh-baseline.json` โ€” fresh JMH results, the new reference' + echo '- `fesod-benchmark/baseline/baseline-meta.json` โ€” provenance (commit, runner, JDK, date)' + echo + echo "Merge this PR to accept the new performance characteristics as the project baseline." + } > "$PR_BODY_FILE" + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -b "$BRANCH" + git add fesod-benchmark/baseline + git commit -m "chore(benchmark): refresh performance baseline from ${{ github.sha }}" + git push origin "$BRANCH" + + gh pr create \ + --base "$BASE_BRANCH" \ + --head "$BRANCH" \ + --title "chore(benchmark): refresh performance baseline" \ + --body-file "$PR_BODY_FILE" diff --git a/fesod-benchmark/baseline/README.md b/fesod-benchmark/baseline/README.md new file mode 100644 index 000000000..a7c03ea68 --- /dev/null +++ b/fesod-benchmark/baseline/README.md @@ -0,0 +1,15 @@ +# Committed performance baseline + +This directory holds the reference results for the performance regression gate +(see [`../benchmark.md`](../benchmark.md)): + +- `jmh-baseline.json` โ€” JMH results of the last accepted `FesodBenchmark` run. Generated by the + *Benchmark* workflow; **must only ever be produced on an `ubuntu-24.04` GitHub runner with JDK 17 + (Temurin)**, because comparisons are absolute. Never commit a baseline generated on a local machine. +- `baseline-meta.json` โ€” provenance of the baseline: source commit, JDK, runner, generation date. + +On the first run after this branch merges, no baseline exists yet: the workflow detects the +bootstrap case, treats every benchmark as NEW (informational, gate passes) and opens a PR that +commits the initial baseline. After that, refresh it via +*Actions โ†’ Benchmark โ†’ Run workflow* with **update_baseline** checked; the workflow opens +a PR containing the new baseline plus a delta report against the previous one. diff --git a/fesod-benchmark/benchmark.md b/fesod-benchmark/benchmark.md new file mode 100644 index 000000000..4bd2986d2 --- /dev/null +++ b/fesod-benchmark/benchmark.md @@ -0,0 +1,147 @@ +# Fesod Benchmark Module + +JMH benchmarks for Fesod spreadsheet operations, plus (per this proposal) a +release-time performance regression gate. Benchmark code is not part of the +Fesod public API. + +It covers three complementary suites (package `org.apache.fesod.sheet.benchmark`): + +| Suite | What it measures | +|---|---| +| `FesodBenchmark` | The core hot paths โ€” **read** and **write**, XLSX and CSV, 1K/10K/100K rows ร— 20 columns, average time per op | +| `FesodVsPoiBenchmark` | Plain-API comparison with **Apache POI** on the same data: write (`ExcelWriter` vs XSSF/HSSF `Workbook`) and read (Fesod streaming reader vs POI's DOM-based `WorkbookFactory`); XLSX and XLS (XLS truncated to 65,534 data rows) | +| `StreamingBenchmark` | Streaming-API comparison for the **large-file** scenario, XLSX only: **read** โ€” Fesod streaming reader vs POI **SAX event model** (`XSSFReader` + `XSSFSheetXMLHandler`); **write** โ€” Fesod batched `ExcelWriter` vs POI **SXSSF** sliding-window workbook; 1K/10K/100K rows | + +Shared support code: `BenchmarkConfiguration` (dataset sizes, file formats), +`data.BenchmarkData` (the 20-column data model), `util.DataGenerator` +(fixed-seed, date-anchored โ€” same seed always produces the same rows) and +`util.BenchmarkFileUtil` (scratch files live under `target/benchmark-testdata`, +which `mvn clean` removes). + +## Building and running + +Build the shaded benchmark jar, then use the standard JMH launcher: + +```bash +./mvnw -B -ntp -pl fesod-benchmark -am package -DskipTests + +# Whole core suite (all sizes ร— formats) +java -jar fesod-benchmark/target/benchmarks.jar FesodBenchmark + +# One slice, e.g. LARGE XLSX reads only +java -jar fesod-benchmark/target/benchmarks.jar 'FesodBenchmark\.read$' \ + -p datasetSize=LARGE -p fileFormat=XLSX + +# Fesod vs POI, XLSX only +java -jar fesod-benchmark/target/benchmarks.jar FesodVsPoiBenchmark -p fileFormat=XLSX + +# Streaming read for the large-file scenario: Fesod vs POI SAX event model +java -jar fesod-benchmark/target/benchmarks.jar StreamingBenchmark + +# Add allocation-per-op (bytes/op) โ€” the most direct memory-efficiency signal +java -jar fesod-benchmark/target/benchmarks.jar FesodBenchmark -prof gc +``` + +Maven shortcuts: + +```bash +# Full run of a suite via the exec plugin +./mvnw -B -ntp -pl fesod-benchmark verify -Pbenchmark '-Dbenchmark.pattern=FesodBenchmark' + +# Quick smoke run (SMALL/CSV, 1 fork, minimal iterations) โ€” sanity check after changes +./mvnw -B -ntp -pl fesod-benchmark verify -Pbenchmark-test +``` + +## Execution contracts + +The annotation contracts are part of each suite and should be kept stable so that +results collected at different times stay comparable: + +| Suite | Mode | Forks | Warmup | Measurement | JVM args | +|---|---|---|---|---|---| +| `FesodBenchmark` | avgt, ms/op | 3 | 3 ร— 1s | 5 ร— 2s | `-Xms1g -Xmx1g -XX:+UseG1GC` | +| `FesodVsPoiBenchmark` | avgt, ms/op | 3 | 3 ร— 5s | 5 ร— 5s | `-Xms2g -Xmx2g` | +| `StreamingBenchmark` | avgt, ms/op | 3 | 3 ร— 5s | 5 ร— 5s | `-Xms2g -Xmx2g` | + +All suites use 3 forks so cross-fork JVM/JIT variability is visible in the +error bars. The core suite deliberately keeps a smaller fixed heap (`1g`, +G1GC) โ€” a roomy constant heap removes GC adaptivity from the timing signal โ€” +while the comparison suites use `2g` so POI's in-memory workloads are not the +constraint being measured. + +## Notes on interpreting results + +- **Environment matters.** Time measurements fluctuate with the host (CPU, + contention, disk). Compare runs from the same machine and similar load, and + trust JMH's error bars (`score ยฑ error`): overlapping intervals between two + runs mean "no observable difference". +- **`-prof gc` allocation per op is the most stable signal** (typically ยฑ0.1% + run-to-run on the same machine). For Fesod's *streaming without OOM* promise, + allocation per op should stay linear in the number of rows and independent of + file size โ€” an accidental full-file buffering shows up here as an + orders-of-magnitude jump long before timing changes. +- **LARGE (100K rows) is where streaming behavior is actually exercised.** At + 1K/10K rows even a non-streaming implementation looks fine; memory + characteristics only separate at larger sizes. +- **Read-path workload symmetry (vs-POI suites).** The write comparisons are + symmetric โ€” both libraries convert the same objects into the same 20 columns. + The read comparisons are deliberately asymmetric, mirroring each library's + idiomatic end-to-end usage: Fesod runs its full pipeline (parse + type + conversion such as `BigDecimal`/dates + reflective row-object mapping into + `BenchmarkData`), while the POI side consumes raw cell text โ€” POI has no + built-in row-object mapping, and hand-rolling one in the benchmark would + measure benchmark code rather than either library. The asymmetry is + conservative for Fesod: the POI side does strictly less work per row, so the + comparison cannot overstate Fesod's advantage. +- **Data is deterministic**: fixed seed and a fixed date anchor + (`2024-01-01`) mean any difference between two runs comes from the code or + the environment, not from the data. + +## Extending + +- New benchmark methods are added like any JMH benchmark (`@Benchmark` method + + `@Param`s). Keep the class-level contract annotations unchanged when adding + to an existing suite; create a new suite class if a different contract is + needed. +- If a benchmark writes files, write them through `BenchmarkFileUtil` so they + land in `target/benchmark-testdata`. Write benchmarks target a fixed file + (overwritten each invocation) and delete it from the trial `@TearDown` via + `BenchmarkFileUtil.delete` โ€” never inside the measured method, so cleanup + I/O stays out of the reported time. + +## Performance regression gate (this PR's proposal) + +On top of the manual suites above, this PR wires `FesodBenchmark` into a +**release-time regression gate**: + +- **Triggers**: release tags (`[0-9]+.*`) and manual `workflow_dispatch` โ€” never + per pull request (shared-runner noise makes every-push gating flaky and + wasteful). Workflow: [`.github/workflows/benchmark.yml`](../.github/workflows/benchmark.yml). +- **Components** (package `org.apache.fesod.sheet.benchmark.baseline`): + `BaselineRunner` runs `FesodBenchmark` with the pinned contract (3 forks, + 3ร—1s warmup, 5ร—2s measurement, `-Xms1g -Xmx1g -XX:+UseG1GC`, gc profiler); + `BaselineComparator` compares against the committed baseline + ([`baseline/`](baseline/)) and renders the Markdown report with the gate verdict. +- **Tiered verdicts** (calibrated against measured noise on real runners): + + | Signal | Noise | Decision | + |---|---|---| + | `gc.alloc.rate.norm` (alloc per op) | ยฑ0.1% | Regression beyond threshold โ†’ **fail** | + | avgt beyond fail threshold **and** non-overlapping JMH error bars | ยฑ7% typical | **Fail** | + | avgt beyond threshold with overlapping error bars | โ€” | **WARN only** | + | Tracked benchmark missing from a run | โ€” | **Fail** | + + Thresholds default to warn 10% / fail 20% (overridable via dispatch inputs or + repo variables `BENCHMARK_WARN_PCT` / `BENCHMARK_FAIL_PCT`). +- **Baseline lifecycle**: the baseline is only ever generated on + `ubuntu-24.04` + JDK 17 Temurin GitHub runners; the first run bootstraps it + via an automated PR, passing tags advance it via automated PRs, and a + regression on a tag posts the report on the matching GitHub Release (or + opens an issue). Benchmark method/param names are the baseline keys โ€” do not + rename them without a baseline refresh. +- **ASF compliance**: only `actions/*` are used; bot pushes go only to + `benchmark/baseline-*` branches; baseline JSON enters the repo exclusively + through reviewed PRs. + +This gate is the part of the original PR #575 that was split out per review +feedback; it is proposed here separately for discussion. diff --git a/fesod-benchmark/pom.xml b/fesod-benchmark/pom.xml new file mode 100644 index 000000000..55bbbde56 --- /dev/null +++ b/fesod-benchmark/pom.xml @@ -0,0 +1,237 @@ + + + + + 4.0.0 + + + org.apache.fesod + fesod-parent + ${revision} + ../pom.xml + + + fesod-benchmark + fesod-benchmark + JMH benchmarks measuring Fesod read/write performance and comparing it with Apache POI + + + 1.37 + 3.1.0 + benchmarks + + + + + org.apache.fesod + fesod-sheet + ${project.version} + + + + + org.openjdk.jmh + jmh-core + ${jmh.version} + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + provided + + + + + org.apache.poi + poi + + + org.apache.poi + poi-ooxml + + + + + org.slf4j + slf4j-api + + + org.slf4j + slf4j-simple + + + + + com.alibaba.fastjson2 + fastjson2 + + + + + org.junit.jupiter + junit-jupiter + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + ${maven.compiler.source} + ${maven.compiler.target} + + + org.projectlombok + lombok + ${lombok.version} + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + ${uberjar.name} + + + org.openjdk.jmh.Main + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + + + + benchmark + + .* + + + + + org.codehaus.mojo + exec-maven-plugin + ${exec-maven-plugin.version} + + + run-benchmarks + integration-test + + java + + + org.openjdk.jmh.Main + + ${benchmark.pattern} + + + + + + + + + + + + benchmark-test + + + + org.codehaus.mojo + exec-maven-plugin + ${exec-maven-plugin.version} + + + run-benchmark-smoke-test + integration-test + + exec + + + java + + -cp + target/benchmarks.jar + org.openjdk.jmh.Main + FesodBenchmark\. + -f + 1 + -wi + 1 + -w + 1s + -i + 2 + -r + 1s + -p + datasetSize=SMALL + -p + fileFormat=CSV + + + + + + + + + + diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/BenchmarkConfiguration.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/BenchmarkConfiguration.java new file mode 100644 index 000000000..084ac0a9d --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/BenchmarkConfiguration.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark; + +import lombok.Getter; + +/** + * Shared constants for the benchmark suites. + */ +public final class BenchmarkConfiguration { + + private BenchmarkConfiguration() {} + + /** + * Dataset sizes used by the benchmark suites. Rows have 20 columns each + * (see {@link org.apache.fesod.sheet.benchmark.data.BenchmarkData}). + */ + @Getter + public enum DatasetSize { + SMALL(1_000, "1K"), + MEDIUM(10_000, "10K"), + LARGE(100_000, "100K"); + + private final int rowCount; + private final String label; + + DatasetSize(int rowCount, String label) { + this.rowCount = rowCount; + this.label = label; + } + } + + /** + * File formats covered by the benchmark suites. Note that XLS is limited + * to 65,536 rows per sheet. + */ + @Getter + public enum FileFormat { + XLSX("xlsx"), + XLS("xls"), + CSV("csv"); + + private final String extension; + + FileFormat(String extension) { + this.extension = extension; + } + } +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/FesodBenchmark.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/FesodBenchmark.java new file mode 100644 index 000000000..c8fd4b9da --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/FesodBenchmark.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark; + +import java.io.File; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.fesod.sheet.ExcelReader; +import org.apache.fesod.sheet.FesodSheet; +import org.apache.fesod.sheet.benchmark.data.BenchmarkData; +import org.apache.fesod.sheet.benchmark.util.BenchmarkFileUtil; +import org.apache.fesod.sheet.benchmark.util.DataGenerator; +import org.apache.fesod.sheet.context.AnalysisContext; +import org.apache.fesod.sheet.read.listener.ReadListener; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Core Fesod benchmark suite: read and write, the two hot paths of the library. + * + * + * + *

The execution contract (forks, warmup, measurement, fixed JVM args) is part of the + * class annotations and should be kept stable so that results collected at different + * times remain comparable. Test data is generated with a fixed seed (see + * {@link DataGenerator}) for reproducibility. Run it through the shaded + * {@code benchmarks.jar} as described in {@code fesod-benchmark/benchmark.md}. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 1) +@Measurement(iterations = 5, time = 2) +@Fork( + value = FesodBenchmark.FORKS, + jvmArgs = {"-Xms1g", "-Xmx1g", "-XX:+UseG1GC"}) +public class FesodBenchmark { + + /** Fork count used when the suite is executed directly through the JMH launcher. */ + public static final int FORKS = 3; + + @Param({"SMALL", "MEDIUM", "LARGE"}) + private String datasetSize; + + @Param({"XLSX", "CSV"}) + private String fileFormat; + + private List data; + private File readFile; + private File writeFile; + private String extension; + + @Setup(Level.Trial) + public void setupTrial() { + BenchmarkConfiguration.DatasetSize size = BenchmarkConfiguration.DatasetSize.valueOf(datasetSize); + extension = BenchmarkConfiguration.FileFormat.valueOf(fileFormat).getExtension(); + data = DataGenerator.generateTestData(size); + + readFile = BenchmarkFileUtil.createTestFile( + String.format("fesod_read_%s_%s.%s", datasetSize.toLowerCase(), extension, extension)); + FesodSheet.write(readFile, BenchmarkData.class).sheet("Sheet1").doWrite(data); + + // Fixed target for the write benchmark: every invocation overwrites the same + // file, and cleanup happens once in tearDownTrial so no I/O other than the + // write itself is part of the measured op. + writeFile = BenchmarkFileUtil.createTestFile( + String.format("fesod_write_%s_%s.%s", datasetSize.toLowerCase(), extension, extension)); + + System.out.printf("FesodBenchmark setup: %s / %s / %d rows%n", fileFormat, datasetSize, data.size()); + } + + @TearDown(Level.Trial) + public void tearDownTrial() { + BenchmarkFileUtil.delete(readFile); + BenchmarkFileUtil.delete(writeFile); + } + + /** + * Write {@code datasetSize} rows of 20 columns to the fixed output file + * (overwritten each invocation, deleted in {@link #tearDownTrial}). + */ + @Benchmark + public long write(Blackhole blackhole) { + FesodSheet.write(writeFile, BenchmarkData.class).sheet("Sheet1").doWrite(data); + blackhole.consume(writeFile.length()); + return data.size(); + } + + /** + * Read the whole pre-generated file through the streaming reader. + */ + @Benchmark + public long read(Blackhole blackhole) { + AtomicLong processedRows = new AtomicLong(0); + + try (ExcelReader excelReader = FesodSheet.read( + readFile, BenchmarkData.class, new ReadListener() { + @Override + public void invoke(BenchmarkData row, AnalysisContext context) { + processedRows.incrementAndGet(); + blackhole.consume(row); + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + // no-op + } + }) + .build()) { + excelReader.readAll(); + } + return processedRows.get(); + } +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/FesodVsPoiBenchmark.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/FesodVsPoiBenchmark.java new file mode 100644 index 000000000..87aa15050 --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/FesodVsPoiBenchmark.java @@ -0,0 +1,302 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.fesod.sheet.ExcelReader; +import org.apache.fesod.sheet.ExcelWriter; +import org.apache.fesod.sheet.FesodSheet; +import org.apache.fesod.sheet.benchmark.data.BenchmarkData; +import org.apache.fesod.sheet.benchmark.util.BenchmarkFileUtil; +import org.apache.fesod.sheet.benchmark.util.DataGenerator; +import org.apache.fesod.sheet.context.AnalysisContext; +import org.apache.fesod.sheet.read.listener.ReadListener; +import org.apache.fesod.sheet.write.metadata.WriteSheet; +import org.apache.poi.hssf.usermodel.HSSFWorkbook; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.ss.usermodel.WorkbookFactory; +import org.apache.poi.util.IOUtils; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Side-by-side comparison of Fesod and Apache POI on the same data, using each + * library's plain (in-memory) APIs: write via {@code ExcelWriter} vs an + * XSSF/HSSF {@code Workbook}, read via the Fesod streaming reader vs POI's + * DOM-based {@code WorkbookFactory}. The streaming-API comparison (POI SAX / + * SXSSF) lives in {@link StreamingBenchmark}. + * + *

Fair-comparison measures applied: + *

+ * + *

Workload symmetry. The write path is symmetric: both sides convert the + * same in-memory objects into the same 20 columns. The read path is deliberately + * asymmetric, mirroring each library's idiomatic end-to-end usage: Fesod runs its + * full pipeline (parse + type conversion such as {@code BigDecimal}/dates + + * reflective {@code BenchmarkData} instantiation), while the POI side consumes raw + * cell text โ€” POI has no built-in row-object mapping, and hand-rolling one in the + * benchmark would measure benchmark code rather than either library. The asymmetry + * is conservative for Fesod: the POI side does strictly less work per row, so the + * comparison cannot overstate Fesod's advantage. + * + *

XLS is limited to 65,536 rows per sheet, so datasets are truncated to 65,534 + * data rows for that format. Run through the shaded {@code benchmarks.jar} as + * described in {@code fesod-benchmark/benchmark.md}. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 5, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 5, timeUnit = TimeUnit.SECONDS) +@Fork( + value = 3, + jvmArgs = {"-Xms2g", "-Xmx2g"}) +public class FesodVsPoiBenchmark { + + /** XLS (BIFF8) supports at most 65,536 rows per sheet; keep one row for the header. */ + private static final int XLS_MAX_DATA_ROWS = 65534; + + @Param({"SMALL", "MEDIUM", "LARGE"}) + private String datasetSize; + + @Param({"XLSX", "XLS"}) + private String fileFormat; + + private File testFile; + private File fesodWriteFile; + private File poiWriteFile; + private List testDataList; + private BenchmarkConfiguration.FileFormat format; + private String extension; + + @Setup(Level.Trial) + public void setupTrial() { + // Allow POI to load the larger files generated by this suite + IOUtils.setByteArrayMaxOverride(1024 * 1024 * 1024); + + format = BenchmarkConfiguration.FileFormat.valueOf(fileFormat); + extension = format.getExtension(); + + BenchmarkConfiguration.DatasetSize size = BenchmarkConfiguration.DatasetSize.valueOf(datasetSize); + testDataList = DataGenerator.generateTestData(size); + + if (format == BenchmarkConfiguration.FileFormat.XLS && testDataList.size() > XLS_MAX_DATA_ROWS) { + System.out.printf( + "WARN: XLS supports max %d data rows, truncating from %d rows.%n", + XLS_MAX_DATA_ROWS, testDataList.size()); + testDataList = new ArrayList<>(testDataList.subList(0, XLS_MAX_DATA_ROWS)); + } + + testFile = BenchmarkFileUtil.createTestFile( + String.format("comparison_%s_%s.%s", datasetSize.toLowerCase(), extension, extension)); + + // Pre-populate the test file used by the read benchmarks + FesodSheet.write(testFile, BenchmarkData.class).sheet("TestData").doWrite(testDataList); + + // Fixed targets for the write benchmarks: every invocation overwrites the + // same file, and cleanup happens once in tearDownTrial so no I/O other + // than the write itself is part of the measured op. + fesodWriteFile = BenchmarkFileUtil.createTestFile( + String.format("fesod_vs_poi_fesod_write_%s.%s", datasetSize.toLowerCase(), extension)); + poiWriteFile = BenchmarkFileUtil.createTestFile( + String.format("fesod_vs_poi_poi_write_%s.%s", datasetSize.toLowerCase(), extension)); + + System.out.printf("FesodVsPoiBenchmark setup: %s format, %d rows%n", fileFormat, testDataList.size()); + } + + @TearDown(Level.Trial) + public void tearDownTrial() { + BenchmarkFileUtil.delete(testFile); + BenchmarkFileUtil.delete(fesodWriteFile); + BenchmarkFileUtil.delete(poiWriteFile); + } + + // ============================================================================ + // WRITE + // ============================================================================ + + /** + * Write the whole dataset with Fesod. + */ + @Benchmark + public long fesodWrite(Blackhole blackhole) { + try (ExcelWriter excelWriter = + FesodSheet.write(fesodWriteFile, BenchmarkData.class).build()) { + WriteSheet writeSheet = FesodSheet.writerSheet("TestData").build(); + excelWriter.write(testDataList, writeSheet); + blackhole.consume(fesodWriteFile.length()); + } + return testDataList.size(); + } + + /** + * Write the same columns with Apache POI. + */ + @Benchmark + public long poiWrite(Blackhole blackhole) throws Exception { + try (FileOutputStream fos = new FileOutputStream(poiWriteFile); + Workbook workbook = createWorkbook()) { + Sheet sheet = workbook.createSheet("TestData"); + + Row headerRow = sheet.createRow(0); + String[] headers = { + "ID", "String Data", "Integer Value", "Long Value", "Double Value", + "BigDecimal Value", "Boolean Flag", "Date Value", "DateTime Value", "Category", + "Description", "Status", "Float Value", "Short Value", "Byte Value", + "Extra Data 1", "Extra Data 2", "Extra Data 3", "Extra Data 4", "Extra Data 5" + }; + for (int i = 0; i < headers.length; i++) { + headerRow.createCell(i).setCellValue(headers[i]); + } + + for (int i = 0; i < testDataList.size(); i++) { + BenchmarkData data = testDataList.get(i); + Row row = sheet.createRow(i + 1); + + row.createCell(0).setCellValue(data.getId() != null ? data.getId() : 0); + row.createCell(1).setCellValue(data.getStringData() != null ? data.getStringData() : ""); + row.createCell(2).setCellValue(data.getIntValue() != null ? data.getIntValue() : 0); + row.createCell(3).setCellValue(data.getLongValue() != null ? data.getLongValue() : 0L); + row.createCell(4).setCellValue(data.getDoubleValue() != null ? data.getDoubleValue() : 0.0); + row.createCell(5) + .setCellValue( + data.getBigDecimalValue() != null + ? data.getBigDecimalValue().doubleValue() + : 0.0); + row.createCell(6).setCellValue(data.getBooleanFlag() != null ? data.getBooleanFlag() : false); + row.createCell(7) + .setCellValue( + data.getDateValue() != null + ? data.getDateValue().toString() + : ""); + row.createCell(8) + .setCellValue( + data.getDateTimeValue() != null + ? data.getDateTimeValue().toString() + : ""); + row.createCell(9).setCellValue(data.getCategory() != null ? data.getCategory() : ""); + row.createCell(10).setCellValue(data.getDescription() != null ? data.getDescription() : ""); + row.createCell(11).setCellValue(data.getStatus() != null ? data.getStatus() : ""); + row.createCell(12).setCellValue(data.getFloatValue() != null ? data.getFloatValue() : 0.0f); + row.createCell(13).setCellValue(data.getShortValue() != null ? data.getShortValue() : 0); + row.createCell(14).setCellValue(data.getByteValue() != null ? data.getByteValue() : 0); + row.createCell(15).setCellValue(data.getExtraData1() != null ? data.getExtraData1() : ""); + row.createCell(16).setCellValue(data.getExtraData2() != null ? data.getExtraData2() : ""); + row.createCell(17).setCellValue(data.getExtraData3() != null ? data.getExtraData3() : ""); + row.createCell(18).setCellValue(data.getExtraData4() != null ? data.getExtraData4() : ""); + row.createCell(19).setCellValue(data.getExtraData5() != null ? data.getExtraData5() : ""); + } + + workbook.write(fos); + blackhole.consume(poiWriteFile.length()); + } + return testDataList.size(); + } + + // ============================================================================ + // READ + // ============================================================================ + + /** + * Stream the whole file with the Fesod reader. + */ + @Benchmark + public long fesodRead(Blackhole blackhole) { + AtomicLong processedRows = new AtomicLong(0); + + try (ExcelReader excelReader = FesodSheet.read( + testFile, BenchmarkData.class, new ReadListener() { + @Override + public void invoke(BenchmarkData data, AnalysisContext context) { + processedRows.incrementAndGet(); + blackhole.consume(data); + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + // no-op + } + }) + .build()) { + excelReader.readAll(); + } + return processedRows.get(); + } + + /** + * Load the whole workbook with Apache POI (the DOM-based approach most POI + * applications use). + */ + @Benchmark + public long poiRead(Blackhole blackhole) throws Exception { + long processedRows = 0; + + try (FileInputStream fis = new FileInputStream(testFile); + Workbook workbook = WorkbookFactory.create(fis)) { + Sheet sheet = workbook.getSheetAt(0); + + for (Row row : sheet) { + if (row.getRowNum() == 0) { + continue; // skip header + } + for (Cell cell : row) { + blackhole.consume(cell.toString()); + } + processedRows++; + } + } + return processedRows; + } + + // ============================================================================ + // UTILITY METHODS + // ============================================================================ + + private Workbook createWorkbook() { + return format == BenchmarkConfiguration.FileFormat.XLSX ? new XSSFWorkbook() : new HSSFWorkbook(); + } +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/StreamingBenchmark.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/StreamingBenchmark.java new file mode 100644 index 000000000..c2466ddbc --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/StreamingBenchmark.java @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.fesod.sheet.ExcelReader; +import org.apache.fesod.sheet.ExcelWriter; +import org.apache.fesod.sheet.FesodSheet; +import org.apache.fesod.sheet.benchmark.data.BenchmarkData; +import org.apache.fesod.sheet.benchmark.util.BenchmarkFileUtil; +import org.apache.fesod.sheet.benchmark.util.DataGenerator; +import org.apache.fesod.sheet.context.AnalysisContext; +import org.apache.fesod.sheet.read.listener.ReadListener; +import org.apache.fesod.sheet.write.metadata.WriteSheet; +import org.apache.poi.openxml4j.opc.OPCPackage; +import org.apache.poi.openxml4j.opc.PackageAccess; +import org.apache.poi.ss.usermodel.DataFormatter; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.util.XMLHelper; +import org.apache.poi.xssf.eventusermodel.ReadOnlySharedStringsTable; +import org.apache.poi.xssf.eventusermodel.XSSFReader; +import org.apache.poi.xssf.eventusermodel.XSSFSheetXMLHandler; +import org.apache.poi.xssf.streaming.SXSSFWorkbook; +import org.apache.poi.xssf.usermodel.XSSFComment; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.xml.sax.ContentHandler; +import org.xml.sax.InputSource; +import org.xml.sax.XMLReader; + +/** + * Streaming-API comparison for the large-file scenario, XLSX only: Fesod vs the + * streaming APIs POI applications use when a workbook must not be fully held in + * memory. + * + *

+ * + *

XLSX only, because the scenario is large files โ€” XLS (BIFF8) tops out at + * 65,536 rows per sheet, and both the SAX event model and SXSSF are XSSF APIs. + * The plain-API comparison lives in {@link FesodVsPoiBenchmark}. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 5, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 5, timeUnit = TimeUnit.SECONDS) +@Fork( + value = 3, + jvmArgs = {"-Xms2g", "-Xmx2g"}) +public class StreamingBenchmark { + + /** Row batch size for the Fesod batched write and the SXSSF window. */ + private static final int BATCH_ROWS = 1000; + + private static final String[] HEADERS = { + "ID", "String Data", "Integer Value", "Long Value", "Double Value", + "BigDecimal Value", "Boolean Flag", "Date Value", "DateTime Value", "Category", + "Description", "Status", "Float Value", "Short Value", "Byte Value", + "Extra Data 1", "Extra Data 2", "Extra Data 3", "Extra Data 4", "Extra Data 5" + }; + + @Param({"FESOD", "POI"}) + private String library; + + @Param({"SMALL", "MEDIUM", "LARGE"}) + private String datasetSize; + + private File testFile; + private File writeFile; + private List testDataList; + + @Setup(Level.Trial) + public void setupTrial() { + BenchmarkConfiguration.DatasetSize size = BenchmarkConfiguration.DatasetSize.valueOf(datasetSize); + testDataList = DataGenerator.generateTestData(size); + + testFile = BenchmarkFileUtil.createTestFile(String.format("streaming_read_%s.xlsx", datasetSize.toLowerCase())); + FesodSheet.write(testFile, BenchmarkData.class).sheet("TestData").doWrite(testDataList); + + // Fixed target for the streaming-write benchmark: overwritten each + // invocation, deleted in tearDownTrial so cleanup stays unmeasured. + writeFile = + BenchmarkFileUtil.createTestFile(String.format("streaming_write_%s.xlsx", datasetSize.toLowerCase())); + + System.out.printf("StreamingBenchmark setup: %s / %s / %d rows%n", library, datasetSize, testDataList.size()); + } + + @TearDown(Level.Trial) + public void tearDownTrial() { + BenchmarkFileUtil.delete(testFile); + BenchmarkFileUtil.delete(writeFile); + } + + @Benchmark + public long read(Blackhole blackhole) throws Exception { + if ("FESOD".equals(library)) { + return fesodRead(blackhole); + } + return poiSaxRead(blackhole); + } + + @Benchmark + public long write(Blackhole blackhole) throws Exception { + if ("FESOD".equals(library)) { + return fesodStreamingWrite(); + } + return poiSxssfWrite(blackhole); + } + + // ============================================================================ + // STREAMING READ + // ============================================================================ + + private long fesodRead(Blackhole blackhole) { + AtomicLong processedRows = new AtomicLong(0); + + try (ExcelReader excelReader = FesodSheet.read( + testFile, BenchmarkData.class, new ReadListener() { + @Override + public void invoke(BenchmarkData row, AnalysisContext context) { + processedRows.incrementAndGet(); + blackhole.consume(row); + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + // no-op + } + }) + .build()) { + excelReader.readAll(); + } + return processedRows.get(); + } + + private long poiSaxRead(Blackhole blackhole) throws Exception { + AtomicLong processedRows = new AtomicLong(0); + + try (OPCPackage pkg = OPCPackage.open(testFile, PackageAccess.READ)) { + ReadOnlySharedStringsTable strings = new ReadOnlySharedStringsTable(pkg); + XSSFReader xssfReader = new XSSFReader(pkg); + XSSFReader.SheetIterator sheets = (XSSFReader.SheetIterator) xssfReader.getSheetsData(); + + while (sheets.hasNext()) { + try (InputStream sheetStream = sheets.next()) { + XSSFSheetXMLHandler.SheetContentsHandler cellHandler = + new XSSFSheetXMLHandler.SheetContentsHandler() { + private int currentRow = -1; + + @Override + public void startRow(int rowNum) { + currentRow = rowNum; + } + + @Override + public void cell(String cellReference, String formattedValue, XSSFComment comment) { + if (currentRow > 0) { + blackhole.consume(formattedValue); + } + } + + @Override + public void endRow(int rowNum) { + if (rowNum > 0) { + processedRows.incrementAndGet(); + } + } + }; + + ContentHandler handler = new XSSFSheetXMLHandler( + xssfReader.getStylesTable(), strings, cellHandler, new DataFormatter(), false); + + XMLReader parser = XMLHelper.newXMLReader(); + parser.setContentHandler(handler); + parser.parse(new InputSource(sheetStream)); + } + } + } + return processedRows.get(); + } + + // ============================================================================ + // STREAMING WRITE + // ============================================================================ + + private long fesodStreamingWrite() { + try (ExcelWriter excelWriter = + FesodSheet.write(writeFile, BenchmarkData.class).build()) { + WriteSheet writeSheet = FesodSheet.writerSheet("TestData").build(); + for (int start = 0; start < testDataList.size(); start += BATCH_ROWS) { + int end = Math.min(start + BATCH_ROWS, testDataList.size()); + excelWriter.write(testDataList.subList(start, end), writeSheet); + } + } + return testDataList.size(); + } + + private long poiSxssfWrite(Blackhole blackhole) throws Exception { + try (SXSSFWorkbook workbook = new SXSSFWorkbook(BATCH_ROWS)) { + Sheet sheet = workbook.createSheet("TestData"); + + Row headerRow = sheet.createRow(0); + for (int i = 0; i < HEADERS.length; i++) { + headerRow.createCell(i).setCellValue(HEADERS[i]); + } + + for (int i = 0; i < testDataList.size(); i++) { + fillRow(sheet.createRow(i + 1), testDataList.get(i)); + } + + try (FileOutputStream fos = new FileOutputStream(writeFile)) { + workbook.write(fos); + } + } // close() also disposes the SXSSF temp files + blackhole.consume(writeFile.length()); + return testDataList.size(); + } + + private void fillRow(Row row, BenchmarkData data) { + row.createCell(0).setCellValue(data.getId() != null ? data.getId() : 0); + row.createCell(1).setCellValue(data.getStringData() != null ? data.getStringData() : ""); + row.createCell(2).setCellValue(data.getIntValue() != null ? data.getIntValue() : 0); + row.createCell(3).setCellValue(data.getLongValue() != null ? data.getLongValue() : 0L); + row.createCell(4).setCellValue(data.getDoubleValue() != null ? data.getDoubleValue() : 0.0); + row.createCell(5) + .setCellValue( + data.getBigDecimalValue() != null + ? data.getBigDecimalValue().doubleValue() + : 0.0); + row.createCell(6).setCellValue(data.getBooleanFlag() != null ? data.getBooleanFlag() : false); + row.createCell(7) + .setCellValue(data.getDateValue() != null ? data.getDateValue().toString() : ""); + row.createCell(8) + .setCellValue( + data.getDateTimeValue() != null + ? data.getDateTimeValue().toString() + : ""); + row.createCell(9).setCellValue(data.getCategory() != null ? data.getCategory() : ""); + row.createCell(10).setCellValue(data.getDescription() != null ? data.getDescription() : ""); + row.createCell(11).setCellValue(data.getStatus() != null ? data.getStatus() : ""); + row.createCell(12).setCellValue(data.getFloatValue() != null ? data.getFloatValue() : 0.0f); + row.createCell(13).setCellValue(data.getShortValue() != null ? data.getShortValue() : 0); + row.createCell(14).setCellValue(data.getByteValue() != null ? data.getByteValue() : 0); + row.createCell(15).setCellValue(data.getExtraData1() != null ? data.getExtraData1() : ""); + row.createCell(16).setCellValue(data.getExtraData2() != null ? data.getExtraData2() : ""); + row.createCell(17).setCellValue(data.getExtraData3() != null ? data.getExtraData3() : ""); + row.createCell(18).setCellValue(data.getExtraData4() != null ? data.getExtraData4() : ""); + row.createCell(19).setCellValue(data.getExtraData5() != null ? data.getExtraData5() : ""); + } +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/baseline/BaselineComparator.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/baseline/BaselineComparator.java new file mode 100644 index 000000000..9f8baeaaf --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/baseline/BaselineComparator.java @@ -0,0 +1,712 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.baseline; + +import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONArray; +import com.alibaba.fastjson2.JSONObject; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.TreeMap; +import java.util.TreeSet; + +/** + * Compares a fresh {@link BaselineRunner} result file against the committed baseline and + * renders a Markdown report for humans (PR comment / job summary), plus a process exit + * code used as the CI gate. + * + *

Comparison rules: + *

+ * + *

Exit codes: {@code 0} = pass (or bootstrap mode without baseline), + * {@code 1} = regression gate failed, {@code 2} = usage or I/O error. + */ +public final class BaselineComparator { + + private static final String GC_ALLOC_RATE_NORM = "gc.alloc.rate.norm"; + + private static final String DEFAULT_BASELINE = "fesod-benchmark/baseline/jmh-baseline.json"; + private static final String DEFAULT_BASELINE_META = "fesod-benchmark/baseline/baseline-meta.json"; + private static final String DEFAULT_CURRENT = "target/baseline-current.json"; + + private static final String OK = ":white_check_mark: OK"; + private static final String IMPROVED = ":green_circle: FASTER"; + private static final String WARN = ":warning: WARN"; + private static final String FAIL = ":red_circle: FAIL"; + private static final String NEW = ":new: NEW"; + private static final String MISSING = ":black_circle: MISSING"; + + private static final String UP = ":small_red_triangle:"; + private static final String DOWN = ":small_red_triangle_down:"; + + private enum Verdict { + OK(0), + IMPROVED(1), + WARN(2), + FAIL(3), + NEW(0), + MISSING(3); + + private final int severity; + + Verdict(int severity) { + this.severity = severity; + } + } + + /** One JMH result entry: primary metric plus the gc allocation metric when present. */ + private static final class Metric { + final String key; + final String displayName; + final String mode; + final double score; + final double scoreError; + final String scoreUnit; + final Double allocScore; + final double allocError; + final String allocUnit; + + Metric( + String key, + String displayName, + String mode, + double score, + double scoreError, + String scoreUnit, + Double allocScore, + double allocError, + String allocUnit) { + this.key = key; + this.displayName = displayName; + this.mode = mode; + this.score = score; + this.scoreError = scoreError; + this.scoreUnit = scoreUnit; + this.allocScore = allocScore; + this.allocError = allocError; + this.allocUnit = allocUnit; + } + } + + /** One report row: baseline/current metric pair plus the computed verdict. */ + private static final class Row { + final Metric baseline; + final Metric current; + Verdict verdict; + String timeDetail; + String allocDetail; + + Row(Metric baseline, Metric current) { + this.baseline = baseline; + this.current = current; + } + + Metric currentOrBaseline() { + return current != null ? current : baseline; + } + } + + private final double warnPct; + private final double failPct; + private final boolean allowMissing; + + private BaselineComparator(double warnPct, double failPct, boolean allowMissing) { + this.warnPct = warnPct; + this.failPct = failPct; + this.allowMissing = allowMissing; + } + + public static void main(String[] args) { + String baselinePath = DEFAULT_BASELINE; + String baselineMetaPath = DEFAULT_BASELINE_META; + String currentPath = DEFAULT_CURRENT; + String reportPath = null; + double warnPct = 10.0d; + double failPct = 20.0d; + boolean allowMissing = false; + + for (int i = 0; i < args.length; i++) { + String arg = args[i]; + if ("--baseline".equals(arg) && i + 1 < args.length) { + baselinePath = args[++i]; + } else if ("--baseline-meta".equals(arg) && i + 1 < args.length) { + baselineMetaPath = args[++i]; + } else if ("--current".equals(arg) && i + 1 < args.length) { + currentPath = args[++i]; + } else if ("--report".equals(arg) && i + 1 < args.length) { + reportPath = args[++i]; + } else if ("--warn".equals(arg) && i + 1 < args.length) { + warnPct = Double.parseDouble(args[++i]); + } else if ("--fail".equals(arg) && i + 1 < args.length) { + failPct = Double.parseDouble(args[++i]); + } else if ("--allow-missing".equals(arg)) { + allowMissing = true; + } else { + usageAndExit("Unknown or incomplete argument: " + arg); + } + } + if (failPct <= warnPct) { + usageAndExit("--fail threshold must be greater than --warn threshold"); + } + + try { + File currentFile = new File(currentPath); + if (!currentFile.isFile()) { + System.err.println("ERROR: current result file not found: " + currentPath); + System.exit(2); + } + Map current = index(readResults(currentFile)); + String currentContract = readContract(currentFile); + + File baselineFile = new File(baselinePath); + Map baseline = + baselineFile.isFile() ? index(readResults(baselineFile)) : new TreeMap(); + String baselineContract = baselineFile.isFile() ? readContract(baselineFile) : null; + boolean contractDiffers = baselineContract != null && !baselineContract.equals(currentContract); + + BaselineComparator comparator = new BaselineComparator(warnPct, failPct, allowMissing); + List rows = comparator.compare(baseline, current); + String report = comparator.renderReport( + rows, + baseline.isEmpty(), + readMeta(new File(baselineMetaPath)), + baselineContract, + currentContract, + contractDiffers); + + System.out.println(); + System.out.println(report); + + if (reportPath != null) { + File reportFile = new File(reportPath); + if (reportFile.getParentFile() != null + && !reportFile.getParentFile().exists()) { + reportFile.getParentFile().mkdirs(); + } + Files.write(reportFile.toPath(), report.getBytes(StandardCharsets.UTF_8)); + System.out.println("Report written to: " + reportPath); + System.out.println(); + } + + boolean gateFailed = false; + for (Row row : rows) { + if (row.verdict == Verdict.FAIL || (row.verdict == Verdict.MISSING && !allowMissing)) { + gateFailed = true; + System.err.println("REGRESSION: " + row.verdict + " " + row.currentOrBaseline().displayName + + " โ€” time: " + row.timeDetail + ", alloc: " + row.allocDetail); + } + } + printAnnotations(rows); + if (contractDiffers) { + System.err.println("NOTE: execution contract differs from baseline โ€” refresh the baseline " + + "for a strictly valid comparison."); + } + + System.exit(gateFailed ? 1 : 0); + } catch (IOException e) { + System.err.println("ERROR: " + e.getMessage()); + System.exit(2); + } + } + + /** + * Emits GitHub Actions workflow commands so regressions surface as annotations on the + * run page (and any linked PR/release), not only in the log. No-op outside Actions. + */ + private static void printAnnotations(List rows) { + if (!"true".equals(System.getenv("GITHUB_ACTIONS"))) { + return; + } + for (Row row : rows) { + String detail = (row.currentOrBaseline().displayName + " โ€” time: " + row.timeDetail + ", alloc: " + + row.allocDetail) + .replace(UP + " ", "") + .replace(DOWN + " ", ""); + if (row.verdict == Verdict.FAIL || (row.verdict == Verdict.MISSING)) { + System.out.println("::error title=Performance regression::" + detail); + } else if (row.verdict == Verdict.WARN) { + System.out.println("::warning title=Performance warning::" + detail); + } + } + } + + // ------------------------------------------------------------------ + // Comparison + // ------------------------------------------------------------------ + + private List compare(Map baseline, Map current) { + List rows = new ArrayList(); + TreeSet keys = new TreeSet(); + keys.addAll(baseline.keySet()); + keys.addAll(current.keySet()); + + for (String key : keys) { + Metric base = baseline.get(key); + Metric cur = current.get(key); + Row row = new Row(base, cur); + + if (base == null) { + row.verdict = Verdict.NEW; + row.timeDetail = "not in baseline"; + row.allocDetail = "n/a"; + } else if (cur == null) { + row.verdict = Verdict.MISSING; + row.timeDetail = "missing from current run"; + row.allocDetail = "n/a"; + } else { + row.verdict = worst(metricVerdict(base, cur), allocVerdict(base, cur)); + row.timeDetail = timeDetail(base, cur); + row.allocDetail = allocDetail(base, cur); + } + rows.add(row); + } + return rows; + } + + private Verdict metricVerdict(Metric base, Metric cur) { + boolean higherIsWorse = higherIsWorse(cur.mode); + return verdict(base.score, base.scoreError, cur.score, cur.scoreError, higherIsWorse); + } + + private Verdict allocVerdict(Metric base, Metric cur) { + if (base.allocScore == null || cur.allocScore == null || base.allocScore <= 0 || cur.allocScore <= 0) { + return Verdict.OK; + } + return verdict(base.allocScore, base.allocError, cur.allocScore, cur.allocError, true); + } + + private Verdict verdict(double base, double baseErr, double cur, double curErr, boolean higherIsWorse) { + double regressionPct = higherIsWorse ? (cur - base) / base * 100.0d : (base - cur) / base * 100.0d; + boolean intervalsOverlap = + higherIsWorse ? (cur - curErr) <= (base + baseErr) : (cur + curErr) >= (base - baseErr); + + if (regressionPct >= failPct && !intervalsOverlap) { + return Verdict.FAIL; + } + if (regressionPct >= failPct || regressionPct >= warnPct) { + return Verdict.WARN; + } + if (regressionPct <= -warnPct) { + return Verdict.IMPROVED; + } + return Verdict.OK; + } + + private String timeDetail(Metric base, Metric cur) { + return changeDetail(cur.score - base.score, base.score); + } + + private String allocDetail(Metric base, Metric cur) { + if (base.allocScore == null || cur.allocScore == null || base.allocScore <= 0) { + return "n/a"; + } + return formatBytes(cur.allocScore) + " (" + changeDetail(cur.allocScore - base.allocScore, base.allocScore) + + ")"; + } + + private static String allocAbsolute(Metric metric) { + return metric == null || metric.allocScore == null ? "โ€”" : formatBytes(metric.allocScore); + } + + private static String changeDetail(double delta, double base) { + double pct = delta / base * 100.0d; + String icon = ""; + if (pct > 0.05d) { + icon = UP + " "; + } else if (pct < -0.05d) { + icon = DOWN + " "; + } + return String.format(Locale.ROOT, "%s%+.1f%%", icon, pct); + } + + private static boolean higherIsWorse(String mode) { + // thrpt: higher score = better; everything else JMH emits (avgt, ss, sampled) is time-like + return !"thrpt".equals(mode); + } + + private static Verdict worst(Verdict a, Verdict b) { + return a.severity >= b.severity ? a : b; + } + + // ------------------------------------------------------------------ + // Report rendering + // ------------------------------------------------------------------ + + private String renderReport( + List rows, + boolean bootstrap, + JSONObject meta, + String baselineContract, + String currentContract, + boolean contractDiffers) { + StringBuilder sb = new StringBuilder(); + + if (bootstrap) { + sb.append("## :bar_chart: Performance run โ€” baseline not yet established\n\n"); + sb.append("No committed baseline found (or it is unreadable), so no comparison was performed.\n"); + sb.append("The CI update job will open a pull request that records this run as the initial baseline.\n"); + sb.append("Every value below becomes a tracked metric once that PR is merged.\n\n"); + } else { + sb.append("## :bar_chart: Performance vs baseline\n\n"); + sb.append(baselineHeader(meta)); + sb.append("\n\n"); + if (contractDiffers) { + sb.append("> :warning: **Execution contract differs from the baseline** โ€” baseline ran `") + .append(baselineContract) + .append("`, this run used `") + .append(currentContract) + .append("`.\n") + .append("> Time deltas across different contracts are indicative only; refresh the ") + .append("baseline to restore a strictly valid comparison.\n\n"); + } + } + + sb.append("| Benchmark | Mode | Baseline | Current | "); + sb.append(bootstrap ? "Alloc/op | Verdict |\n" : "ฮ” time | Alloc/op (ฮ”) | Verdict |\n"); + sb.append(bootstrap ? "|---|:-:|---:|---:|---:|:-:|\n" : "|---|:-:|---:|---:|---:|---:|:-:|\n"); + + for (Row row : rows) { + Metric shown = row.current != null ? row.current : row.baseline; + sb.append("| `") + .append(shown.displayName) + .append("` | ") + .append(shown.mode) + .append(" | ") + .append(row.baseline == null ? "โ€”" : formatScore(row.baseline.score, row.baseline.scoreUnit)) + .append(" | ") + .append(row.current == null ? "โ€”" : formatScore(row.current.score, row.current.scoreUnit)) + .append(" | "); + if (bootstrap) { + sb.append(allocAbsolute(row.current)); + } else { + sb.append(row.timeDetail).append(" | ").append(row.allocDetail); + } + sb.append(" | ").append(verdictLabel(row)).append(" |\n"); + } + + sb.append("\n").append(summaryLine(rows, bootstrap)).append("\n"); + sb.append(legend()); + return sb.toString(); + } + + private String baselineHeader(JSONObject meta) { + StringBuilder sb = new StringBuilder(); + sb.append("Baseline: "); + if (meta == null) { + sb.append("`").append(DEFAULT_BASELINE).append("` (no metadata)"); + } else { + String sha = meta.getString("gitSha"); + sb.append("`") + .append(sha == null ? "unknown" : sha.substring(0, Math.min(7, sha.length()))) + .append("`"); + String generatedAt = meta.getString("generatedAt"); + if (generatedAt != null) { + sb.append(" ยท recorded ").append(generatedAt); + } + String jdk = meta.getString("jdkVersion"); + if (jdk != null) { + sb.append(" ยท ").append(jdk); + } + String runner = meta.getString("runnerLabel"); + if (runner != null) { + sb.append(" ยท ").append(runner); + } + } + sb.append(String.format(Locale.ROOT, " ยท thresholds: warn โ‰ฅ %.0f%% / fail โ‰ฅ %.0f%%", warnPct, failPct)); + return sb.toString(); + } + + private String summaryLine(List rows, boolean bootstrap) { + int ok = 0; + int improved = 0; + int warn = 0; + int fail = 0; + int missing = 0; + int newCount = 0; + for (Row row : rows) { + switch (row.verdict) { + case OK: + ok++; + break; + case IMPROVED: + improved++; + break; + case WARN: + warn++; + break; + case FAIL: + fail++; + break; + case MISSING: + missing++; + break; + case NEW: + newCount++; + break; + default: + break; + } + } + + StringBuilder sb = new StringBuilder("**Summary**: "); + if (bootstrap) { + sb.append(newCount).append(" metric(s) recorded, none compared yet"); + return sb.toString(); + } + sb.append(ok) + .append(" ") + .append(OK) + .append(" ยท ") + .append(improved) + .append(" ") + .append(IMPROVED) + .append(" ยท ") + .append(warn) + .append(" ") + .append(WARN) + .append(" ยท ") + .append(fail) + .append(" ") + .append(FAIL); + if (missing > 0) { + sb.append(" ยท ").append(missing).append(" ").append(MISSING); + } + if (newCount > 0) { + sb.append(" ยท ").append(newCount).append(" ").append(NEW); + } + + boolean gateFailed = fail > 0 || (missing > 0 && !allowMissing); + sb.append(" โ€” gate: **").append(gateFailed ? "FAILED" : "PASSED").append("**"); + if (gateFailed) { + sb.append(" (regression beyond fail threshold or missing benchmark)"); + } + return sb.toString(); + } + + private String legend() { + return "\n

\nVerdict legend & notes\n\n" + + "- " + + OK + + ": within the warn threshold; " + + IMPROVED + + ": at least the warn threshold faster.\n" + + "- " + + WARN + + ": slower beyond the warn threshold, or beyond the fail threshold but still within JMH " + + "statistical error bars (noisy CI runner) โ€” human judgement required.\n" + + "- " + + FAIL + + ": slower beyond the fail threshold with non-overlapping error bars โ€” fails the CI run.\n" + + "- " + + NEW + + ": not tracked by the baseline yet; becomes tracked at the next baseline refresh. " + + MISSING + + ": tracked benchmark absent from the current run.\n" + + "- Time is the JMH average per operation; ฮ” alloc compares `gc.alloc.rate.norm` " + + "(bytes allocated per operation) โ€” a very stable regression signal.\n" + + "- To deliberately accept a performance change, refresh the baseline: " + + "*Actions โ†’ Benchmark โ†’ Run workflow* with **update_baseline** checked " + + "(see `fesod-benchmark/benchmark.md`).\n" + + "\n
\n"; + } + + private static String verdictLabel(Row row) { + switch (row.verdict) { + case IMPROVED: + return IMPROVED; + case WARN: + return WARN; + case FAIL: + return FAIL; + case NEW: + return NEW; + case MISSING: + return MISSING; + default: + return OK; + } + } + + // ------------------------------------------------------------------ + // JMH JSON parsing + // ------------------------------------------------------------------ + + private static List readResults(File file) throws IOException { + String content = new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8); + JSONArray array = JSON.parseArray(content); + List results = new ArrayList(); + for (int i = 0; i < array.size(); i++) { + JSONObject entry = array.getJSONObject(i); + JSONObject primary = entry.getJSONObject("primaryMetric"); + if (primary == null) { + continue; + } + JSONObject params = entry.getJSONObject("params"); + + Double allocScore = null; + double allocError = 0.0d; + String allocUnit = null; + JSONObject secondary = entry.getJSONObject("secondaryMetrics"); + if (secondary != null) { + JSONObject alloc = secondary.getJSONObject(GC_ALLOC_RATE_NORM); + if (alloc != null && alloc.containsKey("score")) { + allocScore = sanitize(alloc.getDouble("score")); + allocError = sanitize(alloc.containsKey("scoreError") ? alloc.getDouble("scoreError") : 0.0d); + allocUnit = alloc.getString("scoreUnit"); + } + } + + String benchmark = entry.getString("benchmark"); + results.add(new Metric( + keyOf(benchmark, params), + displayNameOf(benchmark, params), + entry.getString("mode"), + primary.getDoubleValue("score"), + sanitize(primary.containsKey("scoreError") ? primary.getDouble("scoreError") : 0.0d), + primary.getString("scoreUnit"), + allocScore, + allocError, + allocUnit)); + } + return results; + } + + private static JSONObject readMeta(File file) { + if (!file.isFile()) { + return null; + } + try { + return JSON.parseObject(new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8)); + } catch (IOException e) { + return null; + } + } + + /** + * Reads the JMH execution contract (forks, warmup, measurement) from the first result + * entry, e.g. {@code "3 forks, 3x1 s warmup, 5x2 s measurement"}. Comparisons are only + * strictly valid when the baseline and the current run share the same contract. + */ + private static String readContract(File file) throws IOException { + JSONArray array = JSON.parseArray(new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8)); + if (array.isEmpty()) { + return null; + } + JSONObject entry = array.getJSONObject(0); + return String.format( + Locale.ROOT, + "%d fork%s, %dx%s warmup, %dx%s measurement", + entry.getIntValue("forks"), + entry.getIntValue("forks") == 1 ? "" : "s", + entry.getIntValue("warmupIterations"), + entry.getString("warmupTime"), + entry.getIntValue("measurementIterations"), + entry.getString("measurementTime")); + } + + private static Map index(List metrics) { + Map byKey = new TreeMap(); + for (Metric metric : metrics) { + byKey.put(metric.key, metric); + } + return byKey; + } + + private static String keyOf(String benchmark, JSONObject params) { + return benchmark + paramsSuffix(params); + } + + private static String displayNameOf(String benchmark, JSONObject params) { + String simple = benchmark.substring(benchmark.lastIndexOf('.') + 1); + return simple + paramsSuffix(params); + } + + private static String paramsSuffix(JSONObject params) { + if (params == null || params.isEmpty()) { + return ""; + } + List parts = new ArrayList(); + for (String key : new TreeSet(params.keySet())) { + parts.add(key + "=" + params.getString(key)); + } + return " [" + String.join(", ", parts) + "]"; + } + + /** JMH emits "NaN" errors for short runs; treat those as "no error information". */ + private static double sanitize(Double value) { + if (value == null || value.isNaN() || value.isInfinite()) { + return 0.0d; + } + return value; + } + + /** Human-readable byte size for gc.alloc.rate.norm values (bytes per op). */ + private static String formatBytes(double bytes) { + if (bytes >= 1024.0d * 1024.0d * 1024.0d) { + return String.format(Locale.ROOT, "%.1f GB", bytes / (1024.0d * 1024.0d * 1024.0d)); + } + if (bytes >= 1024.0d * 1024.0d) { + return String.format(Locale.ROOT, "%.1f MB", bytes / (1024.0d * 1024.0d)); + } + if (bytes >= 1024.0d) { + return String.format(Locale.ROOT, "%.0f KB", bytes / 1024.0d); + } + return String.format(Locale.ROOT, "%.0f B", bytes); + } + + private static String formatScore(double score, String unit) { + String number; + double abs = Math.abs(score); + if (abs >= 100.0d) { + number = String.format(Locale.ROOT, "%.0f", score); + } else if (abs >= 10.0d) { + number = String.format(Locale.ROOT, "%.1f", score); + } else { + number = String.format(Locale.ROOT, "%.2f", score); + } + return unit == null || unit.isEmpty() ? number : number + " " + unit; + } + + private static void usageAndExit(String message) { + System.err.println("ERROR: " + message); + System.err.println(); + System.err.println("Usage: BaselineComparator [--baseline ] [--baseline-meta ]"); + System.err.println(" --current [--report ]"); + System.err.println(" [--warn ] [--fail ] [--allow-missing]"); + System.exit(2); + } +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/baseline/BaselineRunner.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/baseline/BaselineRunner.java new file mode 100644 index 000000000..d3c29fdc0 --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/baseline/BaselineRunner.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.baseline; + +import java.io.File; +import org.apache.fesod.sheet.benchmark.FesodBenchmark; +import org.openjdk.jmh.results.format.ResultFormatType; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import org.openjdk.jmh.runner.options.TimeValue; + +/** + * Single entry point for the performance baseline suite used by the regression CI. + * + *

Runs {@link FesodBenchmark} with a fixed execution contract (JVM settings, forks, + * iterations, gc profiler) so that a run recorded today is comparable with the committed + * baseline. Every knob can be overridden with system properties for local experiments, + * but CI always uses the defaults: + * + * 

+ *   benchmark.forks                   (default 3)
+ *   benchmark.warmup.iterations       (default 3)
+ *   benchmark.warmup.seconds          (default 1)
+ *   benchmark.measurement.iterations  (default 5)
+ *   benchmark.measurement.seconds     (default 2)
+ *   benchmark.datasetSizes            (default SMALL,MEDIUM,LARGE)
+ *   benchmark.fileFormats             (default XLSX,CSV)
+ *   benchmark.result                  (default target/baseline-current.json)
+ *
+ * + *

Usage with the shaded jar: + * 

+ *   java -cp fesod-benchmark/target/benchmarks.jar \
+ *       org.apache.fesod.sheet.benchmark.baseline.BaselineRunner
+ *
+ * + *

The gc profiler is always enabled: allocation per operation is far less noisy than + * wall-clock time on shared CI runners and is tracked in the baseline as a second signal. + */ +public final class BaselineRunner { + + private static final String[] FIXED_JVM_ARGS = {"-Xms1g", "-Xmx1g", "-XX:+UseG1GC"}; + + private BaselineRunner() {} + + public static void main(String[] args) throws RunnerException { + int forks = intProperty("benchmark.forks", 3); + int warmupIterations = intProperty("benchmark.warmup.iterations", 3); + int warmupSeconds = intProperty("benchmark.warmup.seconds", 1); + int measurementIterations = intProperty("benchmark.measurement.iterations", 5); + int measurementSeconds = intProperty("benchmark.measurement.seconds", 2); + String[] datasetSizes = + property("benchmark.datasetSizes", "SMALL,MEDIUM,LARGE").split(","); + String[] fileFormats = property("benchmark.fileFormats", "XLSX,CSV").split(","); + String resultFile = property("benchmark.result", "target/baseline-current.json"); + + printEnvironment(forks, warmupIterations, warmupSeconds, measurementIterations, measurementSeconds, resultFile); + + File resultParent = new File(resultFile).getAbsoluteFile().getParentFile(); + if (resultParent != null && !resultParent.exists()) { + resultParent.mkdirs(); + } + + Options opt = new OptionsBuilder() + .include(FesodBenchmark.class.getSimpleName()) + .param("datasetSize", trim(datasetSizes)) + .param("fileFormat", trim(fileFormats)) + .forks(forks) + .warmupIterations(warmupIterations) + .warmupTime(TimeValue.seconds(warmupSeconds)) + .measurementIterations(measurementIterations) + .measurementTime(TimeValue.seconds(measurementSeconds)) + .addProfiler("gc") + .shouldFailOnError(true) + .jvmArgs(FIXED_JVM_ARGS) + .result(resultFile) + .resultFormat(ResultFormatType.JSON) + .build(); + + new Runner(opt).run(); + + System.out.println(); + System.out.println("====================================================="); + System.out.println("Baseline run finished. Results written to: " + resultFile); + System.out.println("Compare with the committed baseline via BaselineComparator."); + System.out.println("====================================================="); + } + + private static void printEnvironment( + int forks, + int warmupIterations, + int warmupSeconds, + int measurementIterations, + int measurementSeconds, + String resultFile) { + System.out.println("====================================================="); + System.out.println("Fesod baseline suite"); + System.out.println(" JVM : " + System.getProperty("java.vm.name") + " " + + System.getProperty("java.version") + " (" + System.getProperty("java.vendor") + ")"); + System.out.println(" OS : " + System.getProperty("os.name") + " " + System.getProperty("os.version") + + " " + System.getProperty("os.arch")); + System.out.println(" Forks : " + forks); + System.out.println(" Warmup : " + warmupIterations + " x " + warmupSeconds + "s"); + System.out.println(" Measurement: " + measurementIterations + " x " + measurementSeconds + "s"); + System.out.println(" JVM args : " + String.join(" ", FIXED_JVM_ARGS)); + System.out.println(" Profiler : gc"); + System.out.println(" Result : " + resultFile); + System.out.println("====================================================="); + } + + private static String[] trim(String[] values) { + String[] trimmed = new String[values.length]; + for (int i = 0; i < values.length; i++) { + trimmed[i] = values[i].trim(); + } + return trimmed; + } + + private static String property(String name, String defaultValue) { + String value = System.getProperty(name); + return value == null || value.trim().isEmpty() ? defaultValue : value.trim(); + } + + private static int intProperty(String name, int defaultValue) { + String value = System.getProperty(name); + if (value == null || value.trim().isEmpty()) { + return defaultValue; + } + try { + return Integer.parseInt(value.trim()); + } catch (NumberFormatException e) { + System.out.println("WARN: cannot parse -D" + name + "=" + value + ", using default " + defaultValue); + return defaultValue; + } + } +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/data/BenchmarkData.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/data/BenchmarkData.java new file mode 100644 index 000000000..a81eb5342 --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/data/BenchmarkData.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.data; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.LocalDateTime; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.fesod.sheet.annotation.ExcelProperty; +import org.apache.fesod.sheet.annotation.format.DateTimeFormat; + +/** + * Standard benchmark data model with various data types for comprehensive testing + */ +@Data +@NoArgsConstructor +public class BenchmarkData { + + @ExcelProperty(value = "ID", index = 0) + private Long id; + + @ExcelProperty(value = "String Data", index = 1) + private String stringData; + + @ExcelProperty(value = "Integer Value", index = 2) + private Integer intValue; + + @ExcelProperty(value = "Long Value", index = 3) + private Long longValue; + + @ExcelProperty(value = "Double Value", index = 4) + private Double doubleValue; + + @ExcelProperty(value = "BigDecimal Value", index = 5) + private BigDecimal bigDecimalValue; + + @ExcelProperty(value = "Boolean Flag", index = 6) + private Boolean booleanFlag; + + @ExcelProperty(value = "Date Value", index = 7) + @DateTimeFormat("yyyy-MM-dd") + private LocalDate dateValue; + + @ExcelProperty(value = "DateTime Value", index = 8) + @DateTimeFormat("yyyy-MM-dd HH:mm:ss") + private LocalDateTime dateTimeValue; + + @ExcelProperty(value = "Category", index = 9) + private String category; + + @ExcelProperty(value = "Description", index = 10) + private String description; + + @ExcelProperty(value = "Status", index = 11) + private String status; + + @ExcelProperty(value = "Float Value", index = 12) + private Float floatValue; + + @ExcelProperty(value = "Short Value", index = 13) + private Short shortValue; + + @ExcelProperty(value = "Byte Value", index = 14) + private Byte byteValue; + + @ExcelProperty(value = "Extra Data 1", index = 15) + private String extraData1; + + @ExcelProperty(value = "Extra Data 2", index = 16) + private String extraData2; + + @ExcelProperty(value = "Extra Data 3", index = 17) + private String extraData3; + + @ExcelProperty(value = "Extra Data 4", index = 18) + private String extraData4; + + @ExcelProperty(value = "Extra Data 5", index = 19) + private String extraData5; +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/util/BenchmarkFileUtil.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/util/BenchmarkFileUtil.java new file mode 100644 index 000000000..0d44bb702 --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/util/BenchmarkFileUtil.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.util; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import lombok.extern.slf4j.Slf4j; + +/** + * Central place for the scratch directory benchmark files are written to + * ({@code target/benchmark-testdata}), so all suites share one location that + * Maven cleans up on {@code mvn clean}. + */ +@Slf4j +public class BenchmarkFileUtil { + + private static final String TEST_DATA_DIR = "target/benchmark-testdata"; + + /** + * Create test data directory if it doesn't exist. + */ + public static void createTestDataDirectory() { + Path testDataPath = Paths.get(TEST_DATA_DIR); + if (!Files.exists(testDataPath)) { + try { + Files.createDirectories(testDataPath); + } catch (IOException e) { + throw new IllegalStateException("Failed to create test data directory: " + testDataPath, e); + } + } + } + + /** + * Create a test file handle with the specified name inside the test data directory. + */ + public static File createTestFile(String fileName) { + createTestDataDirectory(); + return new File(TEST_DATA_DIR, fileName); + } + + /** + * Delete a benchmark scratch file, logging the cause when deletion fails + * instead of failing the run. + */ + public static void delete(File file) { + if (file == null) { + return; + } + try { + Files.deleteIfExists(file.toPath()); + } catch (IOException e) { + log.warn("Failed to delete benchmark file {}: {}", file, e.getMessage()); + } + } +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/util/DataGenerator.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/util/DataGenerator.java new file mode 100644 index 000000000..e5fb2daca --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/util/DataGenerator.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.util; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import lombok.extern.slf4j.Slf4j; +import org.apache.fesod.sheet.benchmark.BenchmarkConfiguration; +import org.apache.fesod.sheet.benchmark.data.BenchmarkData; + +/** + * Generates deterministic benchmark data: a fixed default seed and a fixed date + * anchor mean the same seed always produces the same rows, so benchmark results + * from different runs are comparable. + */ +@Slf4j +public class DataGenerator { + + /** Fixed anchor for generated dates; never uses the wall clock. */ + private static final LocalDate DATE_ANCHOR = LocalDate.of(2024, 1, 1); + + private static final String[] CATEGORIES = { + "Electronics", + "Books", + "Clothing", + "Home & Garden", + "Sports", + "Automotive", + "Health & Beauty", + "Toys & Games", + "Food & Beverage", + "Office Supplies" + }; + + private static final String[] STATUSES = { + "Active", "Inactive", "Pending", "Processing", "Completed", "Cancelled", "On Hold" + }; + + private static final String[] SAMPLE_WORDS = { + "Lorem", + "ipsum", + "dolor", + "sit", + "amet", + "consectetur", + "adipiscing", + "elit", + "sed", + "do", + "eiusmod", + "tempor", + "incididunt", + "ut", + "labore", + "et", + "dolore", + "magna", + "aliqua", + "enim", + "ad", + "minim", + "veniam", + "quis", + "nostrud", + "exercitation", + "ullamco", + "laboris", + "nisi", + "aliquip", + "ex", + "ea", + "commodo" + }; + + private final Random random; + + public DataGenerator() { + this(42L); + } + + public DataGenerator(long seed) { + this.random = new Random(seed); + } + + public List generateData(BenchmarkConfiguration.DatasetSize size) { + return generateData(size.getRowCount()); + } + + public List generateData(int rowCount) { + log.info("Generating {} rows of benchmark data", rowCount); + + List data = new ArrayList<>(rowCount); + long startTime = System.currentTimeMillis(); + + for (int i = 0; i < rowCount; i++) { + data.add(generateSingleRow(i + 1)); + } + + long duration = System.currentTimeMillis() - startTime; + log.info( + "Generated {} rows in {} ms ({} rows/sec)", + rowCount, + duration, + duration > 0 ? rowCount * 1000L / duration : "N/A"); + + return data; + } + + private BenchmarkData generateSingleRow(long id) { + BenchmarkData data = new BenchmarkData(); + + data.setId(id); + data.setStringData(generateRandomString(10, 50)); + data.setIntValue(random.nextInt(1_000_000)); + data.setLongValue(random.nextLong()); + data.setDoubleValue(random.nextDouble() * 1_000_000); + data.setBigDecimalValue( + BigDecimal.valueOf(random.nextDouble() * 1_000_000).setScale(2, RoundingMode.HALF_UP)); + data.setBooleanFlag(random.nextBoolean()); + data.setDateValue(generateRandomDate()); + data.setDateTimeValue(generateRandomDateTime()); + data.setCategory(CATEGORIES[random.nextInt(CATEGORIES.length)]); + data.setDescription(generateRandomDescription()); + data.setStatus(STATUSES[random.nextInt(STATUSES.length)]); + data.setFloatValue(random.nextFloat() * 1000); + data.setShortValue((short) random.nextInt(Short.MAX_VALUE)); + data.setByteValue((byte) random.nextInt(Byte.MAX_VALUE)); + data.setExtraData1(generateRandomString(5, 20)); + data.setExtraData2(generateRandomString(5, 20)); + data.setExtraData3(generateRandomString(5, 20)); + data.setExtraData4(generateRandomString(5, 20)); + data.setExtraData5(generateRandomString(5, 20)); + + return data; + } + + private String generateRandomString(int minLength, int maxLength) { + int length = random.nextInt(maxLength - minLength + 1) + minLength; + StringBuilder sb = new StringBuilder(length); + + for (int i = 0; i < length; i++) { + if (random.nextBoolean()) { + sb.append((char) ('a' + random.nextInt(26))); + } else { + sb.append((char) ('0' + random.nextInt(10))); + } + } + + return sb.toString(); + } + + private String generateRandomDescription() { + int wordCount = random.nextInt(8) + 3; // 3-10 words + StringBuilder sb = new StringBuilder(); + + for (int i = 0; i < wordCount; i++) { + if (i > 0) { + sb.append(" "); + } + sb.append(SAMPLE_WORDS[random.nextInt(SAMPLE_WORDS.length)]); + } + + return sb.toString(); + } + + /** Random date in the five years before {@link #DATE_ANCHOR}. */ + private LocalDate generateRandomDate() { + LocalDate fiveYearsAgo = DATE_ANCHOR.minusYears(5); + long daysBetween = ChronoUnit.DAYS.between(fiveYearsAgo, DATE_ANCHOR); + long randomDays = Math.floorMod(random.nextLong(), daysBetween); + return fiveYearsAgo.plusDays(randomDays); + } + + /** Random datetime in the year before {@link #DATE_ANCHOR}. */ + private LocalDateTime generateRandomDateTime() { + LocalDateTime oneYearAgo = DATE_ANCHOR.atStartOfDay().minusYears(1); + long secondsBetween = ChronoUnit.SECONDS.between(oneYearAgo, DATE_ANCHOR.atStartOfDay()); + long randomSeconds = Math.floorMod(random.nextLong(), secondsBetween); + return oneYearAgo.plusSeconds(randomSeconds); + } + + private static final DataGenerator defaultGenerator = new DataGenerator(42L); + + /** + * Generate test data with the default generator (fixed seed). + */ + public static List generateTestData(BenchmarkConfiguration.DatasetSize size) { + return defaultGenerator.generateData(size); + } +} diff --git a/fesod-benchmark/src/test/java/org/apache/fesod/sheet/benchmark/util/BenchmarkFileUtilTest.java b/fesod-benchmark/src/test/java/org/apache/fesod/sheet/benchmark/util/BenchmarkFileUtilTest.java new file mode 100644 index 000000000..8d35cb8c6 --- /dev/null +++ b/fesod-benchmark/src/test/java/org/apache/fesod/sheet/benchmark/util/BenchmarkFileUtilTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.util; + +import java.io.File; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class BenchmarkFileUtilTest { + + @Test + void createTestFilePlacesFileInTestDataDirectory() { + File file = BenchmarkFileUtil.createTestFile("unit_test_sample.csv"); + + Assertions.assertEquals("unit_test_sample.csv", file.getName()); + Assertions.assertTrue(file.getParentFile().exists()); + Assertions.assertTrue(file.getPath() + .replace(File.separatorChar, '/') + .endsWith("target/benchmark-testdata/unit_test_sample.csv")); + } + + @Test + void createTestDataDirectoryIsIdempotent() { + BenchmarkFileUtil.createTestDataDirectory(); + BenchmarkFileUtil.createTestDataDirectory(); + + File dir = BenchmarkFileUtil.createTestFile("unit_test_dir_check.tmp").getParentFile(); + Assertions.assertTrue(dir.isDirectory()); + } +} diff --git a/fesod-benchmark/src/test/java/org/apache/fesod/sheet/benchmark/util/DataGeneratorTest.java b/fesod-benchmark/src/test/java/org/apache/fesod/sheet/benchmark/util/DataGeneratorTest.java new file mode 100644 index 000000000..e9394a09f --- /dev/null +++ b/fesod-benchmark/src/test/java/org/apache/fesod/sheet/benchmark/util/DataGeneratorTest.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.util; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.fesod.sheet.benchmark.BenchmarkConfiguration; +import org.apache.fesod.sheet.benchmark.data.BenchmarkData; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class DataGeneratorTest { + + private static final int ROWS = 200; + + @Test + void sameSeedProducesIdenticalData() { + List first = new DataGenerator(42L).generateData(ROWS); + List second = new DataGenerator(42L).generateData(ROWS); + + Assertions.assertEquals( + first.stream().map(BenchmarkData::toString).collect(Collectors.toList()), + second.stream().map(BenchmarkData::toString).collect(Collectors.toList())); + } + + @Test + void defaultConstructorUsesFixedSeed() { + List fromDefault = new DataGenerator().generateData(ROWS); + List fromSeed42 = new DataGenerator(42L).generateData(ROWS); + + Assertions.assertEquals( + fromSeed42.stream().map(BenchmarkData::toString).collect(Collectors.toList()), + fromDefault.stream().map(BenchmarkData::toString).collect(Collectors.toList())); + } + + @Test + void differentSeedsProduceDifferentData() { + List first = new DataGenerator(42L) + .generateData(ROWS).stream().map(BenchmarkData::toString).collect(Collectors.toList()); + List second = new DataGenerator(43L) + .generateData(ROWS).stream().map(BenchmarkData::toString).collect(Collectors.toList()); + + Assertions.assertFalse(first.equals(second)); + } + + @Test + void generatedRowsAreComplete() { + List data = new DataGenerator(7L).generateData(ROWS); + + Assertions.assertEquals(ROWS, data.size()); + for (int i = 0; i < data.size(); i++) { + BenchmarkData row = data.get(i); + Assertions.assertEquals(i + 1L, row.getId()); + Assertions.assertFalse(row.getStringData().isEmpty()); + Assertions.assertFalse(row.getCategory().isEmpty()); + Assertions.assertFalse(row.getStatus().isEmpty()); + } + } + + @Test + void datesStayWithinAnchoredWindows() { + List data = new DataGenerator(7L).generateData(ROWS); + + LocalDate dateLowerBound = LocalDate.of(2019, 1, 1); + LocalDate dateUpperBound = LocalDate.of(2024, 1, 1); + LocalDateTime dateTimeLowerBound = LocalDateTime.of(2023, 1, 1, 0, 0); + LocalDateTime dateTimeUpperBound = LocalDateTime.of(2024, 1, 1, 0, 0); + + for (BenchmarkData row : data) { + Assertions.assertTrue(row.getDateValue().isAfter(dateLowerBound.minusDays(1))); + Assertions.assertTrue(row.getDateValue().isBefore(dateUpperBound.plusDays(1))); + Assertions.assertFalse(row.getDateTimeValue().isBefore(dateTimeLowerBound)); + Assertions.assertFalse(row.getDateTimeValue().isAfter(dateTimeUpperBound)); + } + } + + @Test + void generateTestDataMatchesDatasetSize() { + for (BenchmarkConfiguration.DatasetSize size : BenchmarkConfiguration.DatasetSize.values()) { + Assertions.assertEquals( + size.getRowCount(), DataGenerator.generateTestData(size).size()); + } + } +} diff --git a/pom.xml b/pom.xml index 06c6b06d8..a91c9dd46 100644 --- a/pom.xml +++ b/pom.xml @@ -61,6 +61,7 @@ under the License. fesod-common fesod-shaded fesod-sheet + fesod-benchmark