From 15cb569e367dfb47db242ab6e64d19257c5a6a4a Mon Sep 17 00:00:00 2001 From: gongzhongqiang Date: Tue, 6 Oct 2026 18:41:19 +0800 Subject: [PATCH 1/5] feat(benchmark): add JMH benchmark module Adds a fesod-benchmark module with JMH benchmarks for Fesod, kept to pure JMH classes per review feedback: no CI integration, no report or analysis code. Closes #572. Suites (org.apache.fesod.sheet.benchmark): - FesodBenchmark: read/write x XLSX/CSV x 1K/10K/100K rows (average time, 3 forks, fixed 1g G1 heap, stable execution contract) - FesodVsPoiBenchmark: Fesod vs Apache POI on identical data (write, DOM read, batched streaming read; XLSX + XLS) - deterministic data generation (fixed seed + fixed date anchor) and unit tests for the generator and file utilities Benchmarks run manually via the shaded benchmarks.jar; see fesod-benchmark/benchmark.md. --- fesod-benchmark/benchmark.md | 103 ++++++ fesod-benchmark/pom.xml | 226 +++++++++++++ .../benchmark/BenchmarkConfiguration.java | 74 +++++ .../fesod/sheet/benchmark/FesodBenchmark.java | 152 +++++++++ .../sheet/benchmark/FesodVsPoiBenchmark.java | 302 ++++++++++++++++++ .../sheet/benchmark/StreamingBenchmark.java | 298 +++++++++++++++++ .../sheet/benchmark/data/BenchmarkData.java | 283 ++++++++++++++++ .../benchmark/util/BenchmarkFileUtil.java | 77 +++++ .../sheet/benchmark/util/DataGenerator.java | 215 +++++++++++++ .../benchmark/util/BenchmarkFileUtilTest.java | 47 +++ .../benchmark/util/DataGeneratorTest.java | 103 ++++++ pom.xml | 1 + 12 files changed, 1881 insertions(+) create mode 100644 fesod-benchmark/benchmark.md create mode 100644 fesod-benchmark/pom.xml create mode 100644 fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/BenchmarkConfiguration.java create mode 100644 fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/FesodBenchmark.java create mode 100644 fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/FesodVsPoiBenchmark.java create mode 100644 fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/StreamingBenchmark.java create mode 100644 fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/data/BenchmarkData.java create mode 100644 fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/util/BenchmarkFileUtil.java create mode 100644 fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/util/DataGenerator.java create mode 100644 fesod-benchmark/src/test/java/org/apache/fesod/sheet/benchmark/util/BenchmarkFileUtilTest.java create mode 100644 fesod-benchmark/src/test/java/org/apache/fesod/sheet/benchmark/util/DataGeneratorTest.java diff --git a/fesod-benchmark/benchmark.md b/fesod-benchmark/benchmark.md new file mode 100644 index 000000000..fc3b476fa --- /dev/null +++ b/fesod-benchmark/benchmark.md @@ -0,0 +1,103 @@ +# Fesod Benchmark Module + +JMH benchmarks for Fesod spreadsheet operations. This module is a **manual tool**: +it is not wired into any CI workflow and is not part of the Fesod public API. + +It covers three complementary suites (package `org.apache.fesod.sheet.benchmark`): + +| Suite | What it measures | +|---|---| +| `FesodBenchmark` | The core hot paths — **read** and **write**, XLSX and CSV, 1K/10K/100K rows × 20 columns, average time per op | +| `FesodVsPoiBenchmark` | Plain-API comparison with **Apache POI** on the same data: write (`ExcelWriter` vs XSSF/HSSF `Workbook`) and read (Fesod streaming reader vs POI's DOM-based `WorkbookFactory`); XLSX and XLS (XLS truncated to 65,534 data rows) | +| `StreamingBenchmark` | Streaming-API comparison for the **large-file** scenario, XLSX only: **read** — Fesod streaming reader vs POI **SAX event model** (`XSSFReader` + `XSSFSheetXMLHandler`); **write** — Fesod batched `ExcelWriter` vs POI **SXSSF** sliding-window workbook; 1K/10K/100K rows | + +Shared support code: `BenchmarkConfiguration` (dataset sizes, file formats), +`data.BenchmarkData` (the 20-column data model), `util.DataGenerator` +(fixed-seed, date-anchored — same seed always produces the same rows) and +`util.BenchmarkFileUtil` (scratch files live under `target/benchmark-testdata`, +which `mvn clean` removes). + +## Building and running + +Build the shaded benchmark jar, then use the standard JMH launcher: + +```bash +./mvnw -B -ntp -pl fesod-benchmark -am package -DskipTests + +# Whole core suite (all sizes × formats) +java -jar fesod-benchmark/target/benchmarks.jar FesodBenchmark + +# One slice, e.g. LARGE XLSX reads only +java -jar fesod-benchmark/target/benchmarks.jar 'FesodBenchmark\.read$' \ + -p datasetSize=LARGE -p fileFormat=XLSX + +# Fesod vs POI, XLSX only +java -jar fesod-benchmark/target/benchmarks.jar FesodVsPoiBenchmark -p fileFormat=XLSX + +# Streaming read for the large-file scenario: Fesod vs POI SAX event model +java -jar fesod-benchmark/target/benchmarks.jar StreamingBenchmark + +# Add allocation-per-op (bytes/op) — the most direct memory-efficiency signal +java -jar fesod-benchmark/target/benchmarks.jar FesodBenchmark -prof gc +``` + +Maven shortcuts: + +```bash +# Full run of a suite via the exec plugin +./mvnw -B -ntp -pl fesod-benchmark verify -Pbenchmark '-Dbenchmark.pattern=FesodBenchmark' + +# Quick smoke run (SMALL/CSV, 1 fork, minimal iterations) — sanity check after changes +./mvnw -B -ntp -pl fesod-benchmark verify -Pbenchmark-test +``` + +## Execution contracts + +The annotation contracts are part of each suite and should be kept stable so that +results collected at different times stay comparable: + +| Suite | Mode | Forks | Warmup | Measurement | JVM args | +|---|---|---|---|---|---| +| `FesodBenchmark` | avgt, ms/op | 3 | 3 × 1s | 5 × 2s | `-Xms1g -Xmx1g -XX:+UseG1GC` | +| `FesodVsPoiBenchmark` | avgt, ms/op | 1 | 3 × 5s | 5 × 5s | `-Xms2g -Xmx2g` | +| `StreamingBenchmark` | avgt, ms/op | 1 | 3 × 5s | 5 × 5s | `-Xms2g -Xmx2g` | + +## Notes on interpreting results + +- **Environment matters.** Time measurements fluctuate with the host (CPU, + contention, disk). Compare runs from the same machine and similar load, and + trust JMH's error bars (`score ± error`): overlapping intervals between two + runs mean "no observable difference". +- **`-prof gc` allocation per op is the most stable signal** (typically ±0.1% + run-to-run on the same machine). For Fesod's *streaming without OOM* promise, + allocation per op should stay linear in the number of rows and independent of + file size — an accidental full-file buffering shows up here as an + orders-of-magnitude jump long before timing changes. +- **LARGE (100K rows) is where streaming behavior is actually exercised.** At + 1K/10K rows even a non-streaming implementation looks fine; memory + characteristics only separate at larger sizes. +- **Read-path workload symmetry (vs-POI suites).** The write comparisons are + symmetric — both libraries convert the same objects into the same 20 columns. + The read comparisons are deliberately asymmetric, mirroring each library's + idiomatic end-to-end usage: Fesod runs its full pipeline (parse + type + conversion such as `BigDecimal`/dates + reflective row-object mapping into + `BenchmarkData`), while the POI side consumes raw cell text — POI has no + built-in row-object mapping, and hand-rolling one in the benchmark would + measure benchmark code rather than either library. The asymmetry is + conservative for Fesod: the POI side does strictly less work per row, so the + comparison cannot overstate Fesod's advantage. +- **Data is deterministic**: fixed seed and a fixed date anchor + (`2024-01-01`) mean any difference between two runs comes from the code or + the environment, not from the data. + +## Extending + +- New benchmark methods are added like any JMH benchmark (`@Benchmark` method + + `@Param`s). Keep the class-level contract annotations unchanged when adding + to an existing suite; create a new suite class if a different contract is + needed. +- If a benchmark writes files, write them through `BenchmarkFileUtil` so they + land in `target/benchmark-testdata`. Write benchmarks target a fixed file + (overwritten each invocation) and delete it from the trial `@TearDown` via + `BenchmarkFileUtil.delete` — never inside the measured method, so cleanup + I/O stays out of the reported time. diff --git a/fesod-benchmark/pom.xml b/fesod-benchmark/pom.xml new file mode 100644 index 000000000..6ddb3f66a --- /dev/null +++ b/fesod-benchmark/pom.xml @@ -0,0 +1,226 @@ + + + + + 4.0.0 + + + org.apache.fesod + fesod-parent + ${revision} + ../pom.xml + + + fesod-benchmark + fesod-benchmark + JMH benchmarks measuring Fesod read/write performance and comparing it with Apache POI + + + 1.37 + 3.1.0 + benchmarks + + + + + org.apache.fesod + fesod-sheet + ${project.version} + + + + + org.openjdk.jmh + jmh-core + ${jmh.version} + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + provided + + + + + org.apache.poi + poi + + + org.apache.poi + poi-ooxml + + + + + org.slf4j + slf4j-api + + + org.slf4j + slf4j-simple + + + + + org.junit.jupiter + junit-jupiter + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + ${maven.compiler.source} + ${maven.compiler.target} + + + org.openjdk.jmh + jmh-generator-annprocess + ${jmh.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + package + + shade + + + ${uberjar.name} + + + org.openjdk.jmh.Main + + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + + + + + + + + benchmark + + .* + + + + + org.codehaus.mojo + exec-maven-plugin + ${exec-maven-plugin.version} + + + run-benchmarks + integration-test + + java + + + org.openjdk.jmh.Main + + ${benchmark.pattern} + + + + + + + + + + + + benchmark-test + + + + org.codehaus.mojo + exec-maven-plugin + ${exec-maven-plugin.version} + + + run-benchmark-smoke-test + integration-test + + exec + + + java + + -cp + target/benchmarks.jar + org.openjdk.jmh.Main + FesodBenchmark\. + -f + 1 + -wi + 1 + -w + 1s + -i + 2 + -r + 1s + -p + datasetSize=SMALL + -p + fileFormat=CSV + + + + + + + + + + diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/BenchmarkConfiguration.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/BenchmarkConfiguration.java new file mode 100644 index 000000000..8c77524e1 --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/BenchmarkConfiguration.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark; + +/** + * Shared constants for the benchmark suites. + */ +public final class BenchmarkConfiguration { + + private BenchmarkConfiguration() {} + + /** + * Dataset sizes used by the benchmark suites. Rows have 20 columns each + * (see {@link org.apache.fesod.sheet.benchmark.data.BenchmarkData}). + */ + public enum DatasetSize { + SMALL(1_000, "1K"), + MEDIUM(10_000, "10K"), + LARGE(100_000, "100K"); + + private final int rowCount; + private final String label; + + DatasetSize(int rowCount, String label) { + this.rowCount = rowCount; + this.label = label; + } + + public int getRowCount() { + return rowCount; + } + + public String getLabel() { + return label; + } + } + + /** + * File formats covered by the benchmark suites. Note that XLS is limited + * to 65,536 rows per sheet. + */ + public enum FileFormat { + XLSX("xlsx"), + XLS("xls"), + CSV("csv"); + + private final String extension; + + FileFormat(String extension) { + this.extension = extension; + } + + public String getExtension() { + return extension; + } + } +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/FesodBenchmark.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/FesodBenchmark.java new file mode 100644 index 000000000..c8fd4b9da --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/FesodBenchmark.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark; + +import java.io.File; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.fesod.sheet.ExcelReader; +import org.apache.fesod.sheet.FesodSheet; +import org.apache.fesod.sheet.benchmark.data.BenchmarkData; +import org.apache.fesod.sheet.benchmark.util.BenchmarkFileUtil; +import org.apache.fesod.sheet.benchmark.util.DataGenerator; +import org.apache.fesod.sheet.context.AnalysisContext; +import org.apache.fesod.sheet.read.listener.ReadListener; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Core Fesod benchmark suite: read and write, the two hot paths of the library. + * + * + * + *

The execution contract (forks, warmup, measurement, fixed JVM args) is part of the + * class annotations and should be kept stable so that results collected at different + * times remain comparable. Test data is generated with a fixed seed (see + * {@link DataGenerator}) for reproducibility. Run it through the shaded + * {@code benchmarks.jar} as described in {@code fesod-benchmark/benchmark.md}. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 1) +@Measurement(iterations = 5, time = 2) +@Fork( + value = FesodBenchmark.FORKS, + jvmArgs = {"-Xms1g", "-Xmx1g", "-XX:+UseG1GC"}) +public class FesodBenchmark { + + /** Fork count used when the suite is executed directly through the JMH launcher. */ + public static final int FORKS = 3; + + @Param({"SMALL", "MEDIUM", "LARGE"}) + private String datasetSize; + + @Param({"XLSX", "CSV"}) + private String fileFormat; + + private List data; + private File readFile; + private File writeFile; + private String extension; + + @Setup(Level.Trial) + public void setupTrial() { + BenchmarkConfiguration.DatasetSize size = BenchmarkConfiguration.DatasetSize.valueOf(datasetSize); + extension = BenchmarkConfiguration.FileFormat.valueOf(fileFormat).getExtension(); + data = DataGenerator.generateTestData(size); + + readFile = BenchmarkFileUtil.createTestFile( + String.format("fesod_read_%s_%s.%s", datasetSize.toLowerCase(), extension, extension)); + FesodSheet.write(readFile, BenchmarkData.class).sheet("Sheet1").doWrite(data); + + // Fixed target for the write benchmark: every invocation overwrites the same + // file, and cleanup happens once in tearDownTrial so no I/O other than the + // write itself is part of the measured op. + writeFile = BenchmarkFileUtil.createTestFile( + String.format("fesod_write_%s_%s.%s", datasetSize.toLowerCase(), extension, extension)); + + System.out.printf("FesodBenchmark setup: %s / %s / %d rows%n", fileFormat, datasetSize, data.size()); + } + + @TearDown(Level.Trial) + public void tearDownTrial() { + BenchmarkFileUtil.delete(readFile); + BenchmarkFileUtil.delete(writeFile); + } + + /** + * Write {@code datasetSize} rows of 20 columns to the fixed output file + * (overwritten each invocation, deleted in {@link #tearDownTrial}). + */ + @Benchmark + public long write(Blackhole blackhole) { + FesodSheet.write(writeFile, BenchmarkData.class).sheet("Sheet1").doWrite(data); + blackhole.consume(writeFile.length()); + return data.size(); + } + + /** + * Read the whole pre-generated file through the streaming reader. + */ + @Benchmark + public long read(Blackhole blackhole) { + AtomicLong processedRows = new AtomicLong(0); + + try (ExcelReader excelReader = FesodSheet.read( + readFile, BenchmarkData.class, new ReadListener() { + @Override + public void invoke(BenchmarkData row, AnalysisContext context) { + processedRows.incrementAndGet(); + blackhole.consume(row); + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + // no-op + } + }) + .build()) { + excelReader.readAll(); + } + return processedRows.get(); + } +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/FesodVsPoiBenchmark.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/FesodVsPoiBenchmark.java new file mode 100644 index 000000000..7e396504c --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/FesodVsPoiBenchmark.java @@ -0,0 +1,302 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.fesod.sheet.ExcelReader; +import org.apache.fesod.sheet.ExcelWriter; +import org.apache.fesod.sheet.FesodSheet; +import org.apache.fesod.sheet.benchmark.data.BenchmarkData; +import org.apache.fesod.sheet.benchmark.util.BenchmarkFileUtil; +import org.apache.fesod.sheet.benchmark.util.DataGenerator; +import org.apache.fesod.sheet.context.AnalysisContext; +import org.apache.fesod.sheet.read.listener.ReadListener; +import org.apache.fesod.sheet.write.metadata.WriteSheet; +import org.apache.poi.hssf.usermodel.HSSFWorkbook; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.Workbook; +import org.apache.poi.ss.usermodel.WorkbookFactory; +import org.apache.poi.util.IOUtils; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Side-by-side comparison of Fesod and Apache POI on the same data, using each + * library's plain (in-memory) APIs: write via {@code ExcelWriter} vs an + * XSSF/HSSF {@code Workbook}, read via the Fesod streaming reader vs POI's + * DOM-based {@code WorkbookFactory}. The streaming-API comparison (POI SAX / + * SXSSF) lives in {@link StreamingBenchmark}. + * + *

Fair-comparison measures applied: + *

+ * + *

Workload symmetry. The write path is symmetric: both sides convert the + * same in-memory objects into the same 20 columns. The read path is deliberately + * asymmetric, mirroring each library's idiomatic end-to-end usage: Fesod runs its + * full pipeline (parse + type conversion such as {@code BigDecimal}/dates + + * reflective {@code BenchmarkData} instantiation), while the POI side consumes raw + * cell text — POI has no built-in row-object mapping, and hand-rolling one in the + * benchmark would measure benchmark code rather than either library. The asymmetry + * is conservative for Fesod: the POI side does strictly less work per row, so the + * comparison cannot overstate Fesod's advantage. + * + *

XLS is limited to 65,536 rows per sheet, so datasets are truncated to 65,534 + * data rows for that format. Run through the shaded {@code benchmarks.jar} as + * described in {@code fesod-benchmark/benchmark.md}. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 5, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 5, timeUnit = TimeUnit.SECONDS) +@Fork( + value = 1, + jvmArgs = {"-Xms2g", "-Xmx2g"}) +public class FesodVsPoiBenchmark { + + /** XLS (BIFF8) supports at most 65,536 rows per sheet; keep one row for the header. */ + private static final int XLS_MAX_DATA_ROWS = 65534; + + @Param({"SMALL", "MEDIUM", "LARGE"}) + private String datasetSize; + + @Param({"XLSX", "XLS"}) + private String fileFormat; + + private File testFile; + private File fesodWriteFile; + private File poiWriteFile; + private List testDataList; + private BenchmarkConfiguration.FileFormat format; + private String extension; + + @Setup(Level.Trial) + public void setupTrial() { + // Allow POI to load the larger files generated by this suite + IOUtils.setByteArrayMaxOverride(1024 * 1024 * 1024); + + format = BenchmarkConfiguration.FileFormat.valueOf(fileFormat); + extension = format.getExtension(); + + BenchmarkConfiguration.DatasetSize size = BenchmarkConfiguration.DatasetSize.valueOf(datasetSize); + testDataList = DataGenerator.generateTestData(size); + + if (format == BenchmarkConfiguration.FileFormat.XLS && testDataList.size() > XLS_MAX_DATA_ROWS) { + System.out.printf( + "WARN: XLS supports max %d data rows, truncating from %d rows.%n", + XLS_MAX_DATA_ROWS, testDataList.size()); + testDataList = new ArrayList<>(testDataList.subList(0, XLS_MAX_DATA_ROWS)); + } + + testFile = BenchmarkFileUtil.createTestFile( + String.format("comparison_%s_%s.%s", datasetSize.toLowerCase(), extension, extension)); + + // Pre-populate the test file used by the read benchmarks + FesodSheet.write(testFile, BenchmarkData.class).sheet("TestData").doWrite(testDataList); + + // Fixed targets for the write benchmarks: every invocation overwrites the + // same file, and cleanup happens once in tearDownTrial so no I/O other + // than the write itself is part of the measured op. + fesodWriteFile = BenchmarkFileUtil.createTestFile( + String.format("fesod_vs_poi_fesod_write_%s.%s", datasetSize.toLowerCase(), extension)); + poiWriteFile = BenchmarkFileUtil.createTestFile( + String.format("fesod_vs_poi_poi_write_%s.%s", datasetSize.toLowerCase(), extension)); + + System.out.printf("FesodVsPoiBenchmark setup: %s format, %d rows%n", fileFormat, testDataList.size()); + } + + @TearDown(Level.Trial) + public void tearDownTrial() { + BenchmarkFileUtil.delete(testFile); + BenchmarkFileUtil.delete(fesodWriteFile); + BenchmarkFileUtil.delete(poiWriteFile); + } + + // ============================================================================ + // WRITE + // ============================================================================ + + /** + * Write the whole dataset with Fesod. + */ + @Benchmark + public long fesodWrite(Blackhole blackhole) { + try (ExcelWriter excelWriter = + FesodSheet.write(fesodWriteFile, BenchmarkData.class).build()) { + WriteSheet writeSheet = FesodSheet.writerSheet("TestData").build(); + excelWriter.write(testDataList, writeSheet); + blackhole.consume(fesodWriteFile.length()); + } + return testDataList.size(); + } + + /** + * Write the same columns with Apache POI. + */ + @Benchmark + public long poiWrite(Blackhole blackhole) throws Exception { + try (FileOutputStream fos = new FileOutputStream(poiWriteFile); + Workbook workbook = createWorkbook()) { + Sheet sheet = workbook.createSheet("TestData"); + + Row headerRow = sheet.createRow(0); + String[] headers = { + "ID", "String Data", "Integer Value", "Long Value", "Double Value", + "BigDecimal Value", "Boolean Flag", "Date Value", "DateTime Value", "Category", + "Description", "Status", "Float Value", "Short Value", "Byte Value", + "Extra Data 1", "Extra Data 2", "Extra Data 3", "Extra Data 4", "Extra Data 5" + }; + for (int i = 0; i < headers.length; i++) { + headerRow.createCell(i).setCellValue(headers[i]); + } + + for (int i = 0; i < testDataList.size(); i++) { + BenchmarkData data = testDataList.get(i); + Row row = sheet.createRow(i + 1); + + row.createCell(0).setCellValue(data.getId() != null ? data.getId() : 0); + row.createCell(1).setCellValue(data.getStringData() != null ? data.getStringData() : ""); + row.createCell(2).setCellValue(data.getIntValue() != null ? data.getIntValue() : 0); + row.createCell(3).setCellValue(data.getLongValue() != null ? data.getLongValue() : 0L); + row.createCell(4).setCellValue(data.getDoubleValue() != null ? data.getDoubleValue() : 0.0); + row.createCell(5) + .setCellValue( + data.getBigDecimalValue() != null + ? data.getBigDecimalValue().doubleValue() + : 0.0); + row.createCell(6).setCellValue(data.getBooleanFlag() != null ? data.getBooleanFlag() : false); + row.createCell(7) + .setCellValue( + data.getDateValue() != null + ? data.getDateValue().toString() + : ""); + row.createCell(8) + .setCellValue( + data.getDateTimeValue() != null + ? data.getDateTimeValue().toString() + : ""); + row.createCell(9).setCellValue(data.getCategory() != null ? data.getCategory() : ""); + row.createCell(10).setCellValue(data.getDescription() != null ? data.getDescription() : ""); + row.createCell(11).setCellValue(data.getStatus() != null ? data.getStatus() : ""); + row.createCell(12).setCellValue(data.getFloatValue() != null ? data.getFloatValue() : 0.0f); + row.createCell(13).setCellValue(data.getShortValue() != null ? data.getShortValue() : 0); + row.createCell(14).setCellValue(data.getByteValue() != null ? data.getByteValue() : 0); + row.createCell(15).setCellValue(data.getExtraData1() != null ? data.getExtraData1() : ""); + row.createCell(16).setCellValue(data.getExtraData2() != null ? data.getExtraData2() : ""); + row.createCell(17).setCellValue(data.getExtraData3() != null ? data.getExtraData3() : ""); + row.createCell(18).setCellValue(data.getExtraData4() != null ? data.getExtraData4() : ""); + row.createCell(19).setCellValue(data.getExtraData5() != null ? data.getExtraData5() : ""); + } + + workbook.write(fos); + blackhole.consume(poiWriteFile.length()); + } + return testDataList.size(); + } + + // ============================================================================ + // READ + // ============================================================================ + + /** + * Stream the whole file with the Fesod reader. + */ + @Benchmark + public long fesodRead(Blackhole blackhole) { + AtomicLong processedRows = new AtomicLong(0); + + try (ExcelReader excelReader = FesodSheet.read( + testFile, BenchmarkData.class, new ReadListener() { + @Override + public void invoke(BenchmarkData data, AnalysisContext context) { + processedRows.incrementAndGet(); + blackhole.consume(data); + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + // no-op + } + }) + .build()) { + excelReader.readAll(); + } + return processedRows.get(); + } + + /** + * Load the whole workbook with Apache POI (the DOM-based approach most POI + * applications use). + */ + @Benchmark + public long poiRead(Blackhole blackhole) throws Exception { + long processedRows = 0; + + try (FileInputStream fis = new FileInputStream(testFile); + Workbook workbook = WorkbookFactory.create(fis)) { + Sheet sheet = workbook.getSheetAt(0); + + for (Row row : sheet) { + if (row.getRowNum() == 0) { + continue; // skip header + } + for (Cell cell : row) { + blackhole.consume(cell.toString()); + } + processedRows++; + } + } + return processedRows; + } + + // ============================================================================ + // UTILITY METHODS + // ============================================================================ + + private Workbook createWorkbook() { + return format == BenchmarkConfiguration.FileFormat.XLSX ? new XSSFWorkbook() : new HSSFWorkbook(); + } +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/StreamingBenchmark.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/StreamingBenchmark.java new file mode 100644 index 000000000..9b45975b6 --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/StreamingBenchmark.java @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.fesod.sheet.ExcelReader; +import org.apache.fesod.sheet.ExcelWriter; +import org.apache.fesod.sheet.FesodSheet; +import org.apache.fesod.sheet.benchmark.data.BenchmarkData; +import org.apache.fesod.sheet.benchmark.util.BenchmarkFileUtil; +import org.apache.fesod.sheet.benchmark.util.DataGenerator; +import org.apache.fesod.sheet.context.AnalysisContext; +import org.apache.fesod.sheet.read.listener.ReadListener; +import org.apache.fesod.sheet.write.metadata.WriteSheet; +import org.apache.poi.openxml4j.opc.OPCPackage; +import org.apache.poi.openxml4j.opc.PackageAccess; +import org.apache.poi.ss.usermodel.DataFormatter; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.util.XMLHelper; +import org.apache.poi.xssf.eventusermodel.ReadOnlySharedStringsTable; +import org.apache.poi.xssf.eventusermodel.XSSFReader; +import org.apache.poi.xssf.eventusermodel.XSSFSheetXMLHandler; +import org.apache.poi.xssf.streaming.SXSSFWorkbook; +import org.apache.poi.xssf.usermodel.XSSFComment; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; +import org.xml.sax.ContentHandler; +import org.xml.sax.InputSource; +import org.xml.sax.XMLReader; + +/** + * Streaming-API comparison for the large-file scenario, XLSX only: Fesod vs the + * streaming APIs POI applications use when a workbook must not be fully held in + * memory. + * + *

+ * + *

XLSX only, because the scenario is large files — XLS (BIFF8) tops out at + * 65,536 rows per sheet, and both the SAX event model and SXSSF are XSSF APIs. + * The plain-API comparison lives in {@link FesodVsPoiBenchmark}. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +@Warmup(iterations = 3, time = 5, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 5, time = 5, timeUnit = TimeUnit.SECONDS) +@Fork( + value = 1, + jvmArgs = {"-Xms2g", "-Xmx2g"}) +public class StreamingBenchmark { + + /** Row batch size for the Fesod batched write and the SXSSF window. */ + private static final int BATCH_ROWS = 1000; + + private static final String[] HEADERS = { + "ID", "String Data", "Integer Value", "Long Value", "Double Value", + "BigDecimal Value", "Boolean Flag", "Date Value", "DateTime Value", "Category", + "Description", "Status", "Float Value", "Short Value", "Byte Value", + "Extra Data 1", "Extra Data 2", "Extra Data 3", "Extra Data 4", "Extra Data 5" + }; + + @Param({"FESOD", "POI"}) + private String library; + + @Param({"SMALL", "MEDIUM", "LARGE"}) + private String datasetSize; + + private File testFile; + private File writeFile; + private List testDataList; + + @Setup(Level.Trial) + public void setupTrial() { + BenchmarkConfiguration.DatasetSize size = BenchmarkConfiguration.DatasetSize.valueOf(datasetSize); + testDataList = DataGenerator.generateTestData(size); + + testFile = BenchmarkFileUtil.createTestFile(String.format("streaming_read_%s.xlsx", datasetSize.toLowerCase())); + FesodSheet.write(testFile, BenchmarkData.class).sheet("TestData").doWrite(testDataList); + + // Fixed target for the streaming-write benchmark: overwritten each + // invocation, deleted in tearDownTrial so cleanup stays unmeasured. + writeFile = + BenchmarkFileUtil.createTestFile(String.format("streaming_write_%s.xlsx", datasetSize.toLowerCase())); + + System.out.printf("StreamingBenchmark setup: %s / %s / %d rows%n", library, datasetSize, testDataList.size()); + } + + @TearDown(Level.Trial) + public void tearDownTrial() { + BenchmarkFileUtil.delete(testFile); + BenchmarkFileUtil.delete(writeFile); + } + + @Benchmark + public long read(Blackhole blackhole) throws Exception { + if ("FESOD".equals(library)) { + return fesodRead(blackhole); + } + return poiSaxRead(blackhole); + } + + @Benchmark + public long write(Blackhole blackhole) throws Exception { + if ("FESOD".equals(library)) { + return fesodStreamingWrite(); + } + return poiSxssfWrite(blackhole); + } + + // ============================================================================ + // STREAMING READ + // ============================================================================ + + private long fesodRead(Blackhole blackhole) { + AtomicLong processedRows = new AtomicLong(0); + + try (ExcelReader excelReader = FesodSheet.read( + testFile, BenchmarkData.class, new ReadListener() { + @Override + public void invoke(BenchmarkData row, AnalysisContext context) { + processedRows.incrementAndGet(); + blackhole.consume(row); + } + + @Override + public void doAfterAllAnalysed(AnalysisContext context) { + // no-op + } + }) + .build()) { + excelReader.readAll(); + } + return processedRows.get(); + } + + private long poiSaxRead(Blackhole blackhole) throws Exception { + AtomicLong processedRows = new AtomicLong(0); + + try (OPCPackage pkg = OPCPackage.open(testFile, PackageAccess.READ)) { + ReadOnlySharedStringsTable strings = new ReadOnlySharedStringsTable(pkg); + XSSFReader xssfReader = new XSSFReader(pkg); + XSSFReader.SheetIterator sheets = (XSSFReader.SheetIterator) xssfReader.getSheetsData(); + + while (sheets.hasNext()) { + try (InputStream sheetStream = sheets.next()) { + XSSFSheetXMLHandler.SheetContentsHandler cellHandler = + new XSSFSheetXMLHandler.SheetContentsHandler() { + private int currentRow = -1; + + @Override + public void startRow(int rowNum) { + currentRow = rowNum; + } + + @Override + public void cell(String cellReference, String formattedValue, XSSFComment comment) { + if (currentRow > 0) { + blackhole.consume(formattedValue); + } + } + + @Override + public void endRow(int rowNum) { + if (rowNum > 0) { + processedRows.incrementAndGet(); + } + } + }; + + ContentHandler handler = new XSSFSheetXMLHandler( + xssfReader.getStylesTable(), strings, cellHandler, new DataFormatter(), false); + + XMLReader parser = XMLHelper.newXMLReader(); + parser.setContentHandler(handler); + parser.parse(new InputSource(sheetStream)); + } + } + } + return processedRows.get(); + } + + // ============================================================================ + // STREAMING WRITE + // ============================================================================ + + private long fesodStreamingWrite() { + try (ExcelWriter excelWriter = + FesodSheet.write(writeFile, BenchmarkData.class).build()) { + WriteSheet writeSheet = FesodSheet.writerSheet("TestData").build(); + for (int start = 0; start < testDataList.size(); start += BATCH_ROWS) { + int end = Math.min(start + BATCH_ROWS, testDataList.size()); + excelWriter.write(testDataList.subList(start, end), writeSheet); + } + } + return testDataList.size(); + } + + private long poiSxssfWrite(Blackhole blackhole) throws Exception { + try (SXSSFWorkbook workbook = new SXSSFWorkbook(BATCH_ROWS)) { + Sheet sheet = workbook.createSheet("TestData"); + + Row headerRow = sheet.createRow(0); + for (int i = 0; i < HEADERS.length; i++) { + headerRow.createCell(i).setCellValue(HEADERS[i]); + } + + for (int i = 0; i < testDataList.size(); i++) { + fillRow(sheet.createRow(i + 1), testDataList.get(i)); + } + + try (FileOutputStream fos = new FileOutputStream(writeFile)) { + workbook.write(fos); + } + } // close() also disposes the SXSSF temp files + blackhole.consume(writeFile.length()); + return testDataList.size(); + } + + private void fillRow(Row row, BenchmarkData data) { + row.createCell(0).setCellValue(data.getId() != null ? data.getId() : 0); + row.createCell(1).setCellValue(data.getStringData() != null ? data.getStringData() : ""); + row.createCell(2).setCellValue(data.getIntValue() != null ? data.getIntValue() : 0); + row.createCell(3).setCellValue(data.getLongValue() != null ? data.getLongValue() : 0L); + row.createCell(4).setCellValue(data.getDoubleValue() != null ? data.getDoubleValue() : 0.0); + row.createCell(5) + .setCellValue( + data.getBigDecimalValue() != null + ? data.getBigDecimalValue().doubleValue() + : 0.0); + row.createCell(6).setCellValue(data.getBooleanFlag() != null ? data.getBooleanFlag() : false); + row.createCell(7) + .setCellValue(data.getDateValue() != null ? data.getDateValue().toString() : ""); + row.createCell(8) + .setCellValue( + data.getDateTimeValue() != null + ? data.getDateTimeValue().toString() + : ""); + row.createCell(9).setCellValue(data.getCategory() != null ? data.getCategory() : ""); + row.createCell(10).setCellValue(data.getDescription() != null ? data.getDescription() : ""); + row.createCell(11).setCellValue(data.getStatus() != null ? data.getStatus() : ""); + row.createCell(12).setCellValue(data.getFloatValue() != null ? data.getFloatValue() : 0.0f); + row.createCell(13).setCellValue(data.getShortValue() != null ? data.getShortValue() : 0); + row.createCell(14).setCellValue(data.getByteValue() != null ? data.getByteValue() : 0); + row.createCell(15).setCellValue(data.getExtraData1() != null ? data.getExtraData1() : ""); + row.createCell(16).setCellValue(data.getExtraData2() != null ? data.getExtraData2() : ""); + row.createCell(17).setCellValue(data.getExtraData3() != null ? data.getExtraData3() : ""); + row.createCell(18).setCellValue(data.getExtraData4() != null ? data.getExtraData4() : ""); + row.createCell(19).setCellValue(data.getExtraData5() != null ? data.getExtraData5() : ""); + } +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/data/BenchmarkData.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/data/BenchmarkData.java new file mode 100644 index 000000000..174339ee4 --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/data/BenchmarkData.java @@ -0,0 +1,283 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.data; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.LocalDateTime; +import org.apache.fesod.sheet.annotation.ExcelProperty; +import org.apache.fesod.sheet.annotation.format.DateTimeFormat; + +/** + * Standard benchmark data model with various data types for comprehensive testing + */ +public class BenchmarkData { + + @ExcelProperty(value = "ID", index = 0) + private Long id; + + @ExcelProperty(value = "String Data", index = 1) + private String stringData; + + @ExcelProperty(value = "Integer Value", index = 2) + private Integer intValue; + + @ExcelProperty(value = "Long Value", index = 3) + private Long longValue; + + @ExcelProperty(value = "Double Value", index = 4) + private Double doubleValue; + + @ExcelProperty(value = "BigDecimal Value", index = 5) + private BigDecimal bigDecimalValue; + + @ExcelProperty(value = "Boolean Flag", index = 6) + private Boolean booleanFlag; + + @ExcelProperty(value = "Date Value", index = 7) + @DateTimeFormat("yyyy-MM-dd") + private LocalDate dateValue; + + @ExcelProperty(value = "DateTime Value", index = 8) + @DateTimeFormat("yyyy-MM-dd HH:mm:ss") + private LocalDateTime dateTimeValue; + + @ExcelProperty(value = "Category", index = 9) + private String category; + + @ExcelProperty(value = "Description", index = 10) + private String description; + + @ExcelProperty(value = "Status", index = 11) + private String status; + + @ExcelProperty(value = "Float Value", index = 12) + private Float floatValue; + + @ExcelProperty(value = "Short Value", index = 13) + private Short shortValue; + + @ExcelProperty(value = "Byte Value", index = 14) + private Byte byteValue; + + @ExcelProperty(value = "Extra Data 1", index = 15) + private String extraData1; + + @ExcelProperty(value = "Extra Data 2", index = 16) + private String extraData2; + + @ExcelProperty(value = "Extra Data 3", index = 17) + private String extraData3; + + @ExcelProperty(value = "Extra Data 4", index = 18) + private String extraData4; + + @ExcelProperty(value = "Extra Data 5", index = 19) + private String extraData5; + + // Default constructor + public BenchmarkData() {} + + // Getters and setters + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getStringData() { + return stringData; + } + + public void setStringData(String stringData) { + this.stringData = stringData; + } + + public Integer getIntValue() { + return intValue; + } + + public void setIntValue(Integer intValue) { + this.intValue = intValue; + } + + public Long getLongValue() { + return longValue; + } + + public void setLongValue(Long longValue) { + this.longValue = longValue; + } + + public Double getDoubleValue() { + return doubleValue; + } + + public void setDoubleValue(Double doubleValue) { + this.doubleValue = doubleValue; + } + + public BigDecimal getBigDecimalValue() { + return bigDecimalValue; + } + + public void setBigDecimalValue(BigDecimal bigDecimalValue) { + this.bigDecimalValue = bigDecimalValue; + } + + public Boolean getBooleanFlag() { + return booleanFlag; + } + + public void setBooleanFlag(Boolean booleanFlag) { + this.booleanFlag = booleanFlag; + } + + public LocalDate getDateValue() { + return dateValue; + } + + public void setDateValue(LocalDate dateValue) { + this.dateValue = dateValue; + } + + public LocalDateTime getDateTimeValue() { + return dateTimeValue; + } + + public void setDateTimeValue(LocalDateTime dateTimeValue) { + this.dateTimeValue = dateTimeValue; + } + + public String getCategory() { + return category; + } + + public void setCategory(String category) { + this.category = category; + } + + public String getDescription() { + return description; + } + + public void setDescription(String description) { + this.description = description; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public Float getFloatValue() { + return floatValue; + } + + public void setFloatValue(Float floatValue) { + this.floatValue = floatValue; + } + + public Short getShortValue() { + return shortValue; + } + + public void setShortValue(Short shortValue) { + this.shortValue = shortValue; + } + + public Byte getByteValue() { + return byteValue; + } + + public void setByteValue(Byte byteValue) { + this.byteValue = byteValue; + } + + public String getExtraData1() { + return extraData1; + } + + public void setExtraData1(String extraData1) { + this.extraData1 = extraData1; + } + + public String getExtraData2() { + return extraData2; + } + + public void setExtraData2(String extraData2) { + this.extraData2 = extraData2; + } + + public String getExtraData3() { + return extraData3; + } + + public void setExtraData3(String extraData3) { + this.extraData3 = extraData3; + } + + public String getExtraData4() { + return extraData4; + } + + public void setExtraData4(String extraData4) { + this.extraData4 = extraData4; + } + + public String getExtraData5() { + return extraData5; + } + + public void setExtraData5(String extraData5) { + this.extraData5 = extraData5; + } + + @Override + public String toString() { + return "BenchmarkData{" + "id=" + + id + ", stringData='" + + stringData + '\'' + ", intValue=" + + intValue + ", longValue=" + + longValue + ", doubleValue=" + + doubleValue + ", bigDecimalValue=" + + bigDecimalValue + ", booleanFlag=" + + booleanFlag + ", dateValue=" + + dateValue + ", dateTimeValue=" + + dateTimeValue + ", category='" + + category + '\'' + ", description='" + + description + '\'' + ", status='" + + status + '\'' + ", floatValue=" + + floatValue + ", shortValue=" + + shortValue + ", byteValue=" + + byteValue + ", extraData1='" + + extraData1 + '\'' + ", extraData2='" + + extraData2 + '\'' + ", extraData3='" + + extraData3 + '\'' + ", extraData4='" + + extraData4 + '\'' + ", extraData5='" + + extraData5 + '\'' + '}'; + } +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/util/BenchmarkFileUtil.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/util/BenchmarkFileUtil.java new file mode 100644 index 000000000..cd3a09f4b --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/util/BenchmarkFileUtil.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.util; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Central place for the scratch directory benchmark files are written to + * ({@code target/benchmark-testdata}), so all suites share one location that + * Maven cleans up on {@code mvn clean}. + */ +public class BenchmarkFileUtil { + + private static final Logger logger = LoggerFactory.getLogger(BenchmarkFileUtil.class); + + private static final String TEST_DATA_DIR = "target/benchmark-testdata"; + + /** + * Create test data directory if it doesn't exist. + */ + public static void createTestDataDirectory() { + Path testDataPath = Paths.get(TEST_DATA_DIR); + if (!Files.exists(testDataPath)) { + try { + Files.createDirectories(testDataPath); + } catch (IOException e) { + throw new IllegalStateException("Failed to create test data directory: " + testDataPath, e); + } + } + } + + /** + * Create a test file handle with the specified name inside the test data directory. + */ + public static File createTestFile(String fileName) { + createTestDataDirectory(); + return new File(TEST_DATA_DIR, fileName); + } + + /** + * Delete a benchmark scratch file, logging the cause when deletion fails + * instead of failing the run. + */ + public static void delete(File file) { + if (file == null) { + return; + } + try { + Files.deleteIfExists(file.toPath()); + } catch (IOException e) { + logger.warn("Failed to delete benchmark file {}: {}", file, e.getMessage()); + } + } +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/util/DataGenerator.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/util/DataGenerator.java new file mode 100644 index 000000000..9cab6ed5e --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/util/DataGenerator.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.util; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import org.apache.fesod.sheet.benchmark.BenchmarkConfiguration; +import org.apache.fesod.sheet.benchmark.data.BenchmarkData; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Generates deterministic benchmark data: a fixed default seed and a fixed date + * anchor mean the same seed always produces the same rows, so benchmark results + * from different runs are comparable. + */ +public class DataGenerator { + + private static final Logger logger = LoggerFactory.getLogger(DataGenerator.class); + + /** Fixed anchor for generated dates; never uses the wall clock. */ + private static final LocalDate DATE_ANCHOR = LocalDate.of(2024, 1, 1); + + private static final String[] CATEGORIES = { + "Electronics", + "Books", + "Clothing", + "Home & Garden", + "Sports", + "Automotive", + "Health & Beauty", + "Toys & Games", + "Food & Beverage", + "Office Supplies" + }; + + private static final String[] STATUSES = { + "Active", "Inactive", "Pending", "Processing", "Completed", "Cancelled", "On Hold" + }; + + private static final String[] SAMPLE_WORDS = { + "Lorem", + "ipsum", + "dolor", + "sit", + "amet", + "consectetur", + "adipiscing", + "elit", + "sed", + "do", + "eiusmod", + "tempor", + "incididunt", + "ut", + "labore", + "et", + "dolore", + "magna", + "aliqua", + "enim", + "ad", + "minim", + "veniam", + "quis", + "nostrud", + "exercitation", + "ullamco", + "laboris", + "nisi", + "aliquip", + "ex", + "ea", + "commodo" + }; + + private final Random random; + + public DataGenerator() { + this(42L); + } + + public DataGenerator(long seed) { + this.random = new Random(seed); + } + + public List generateData(BenchmarkConfiguration.DatasetSize size) { + return generateData(size.getRowCount()); + } + + public List generateData(int rowCount) { + logger.info("Generating {} rows of benchmark data", rowCount); + + List data = new ArrayList<>(rowCount); + long startTime = System.currentTimeMillis(); + + for (int i = 0; i < rowCount; i++) { + data.add(generateSingleRow(i + 1)); + } + + long duration = System.currentTimeMillis() - startTime; + logger.info( + "Generated {} rows in {} ms ({} rows/sec)", + rowCount, + duration, + duration > 0 ? rowCount * 1000L / duration : "N/A"); + + return data; + } + + private BenchmarkData generateSingleRow(long id) { + BenchmarkData data = new BenchmarkData(); + + data.setId(id); + data.setStringData(generateRandomString(10, 50)); + data.setIntValue(random.nextInt(1_000_000)); + data.setLongValue(random.nextLong()); + data.setDoubleValue(random.nextDouble() * 1_000_000); + data.setBigDecimalValue( + BigDecimal.valueOf(random.nextDouble() * 1_000_000).setScale(2, RoundingMode.HALF_UP)); + data.setBooleanFlag(random.nextBoolean()); + data.setDateValue(generateRandomDate()); + data.setDateTimeValue(generateRandomDateTime()); + data.setCategory(CATEGORIES[random.nextInt(CATEGORIES.length)]); + data.setDescription(generateRandomDescription()); + data.setStatus(STATUSES[random.nextInt(STATUSES.length)]); + data.setFloatValue(random.nextFloat() * 1000); + data.setShortValue((short) random.nextInt(Short.MAX_VALUE)); + data.setByteValue((byte) random.nextInt(Byte.MAX_VALUE)); + data.setExtraData1(generateRandomString(5, 20)); + data.setExtraData2(generateRandomString(5, 20)); + data.setExtraData3(generateRandomString(5, 20)); + data.setExtraData4(generateRandomString(5, 20)); + data.setExtraData5(generateRandomString(5, 20)); + + return data; + } + + private String generateRandomString(int minLength, int maxLength) { + int length = random.nextInt(maxLength - minLength + 1) + minLength; + StringBuilder sb = new StringBuilder(length); + + for (int i = 0; i < length; i++) { + if (random.nextBoolean()) { + sb.append((char) ('a' + random.nextInt(26))); + } else { + sb.append((char) ('0' + random.nextInt(10))); + } + } + + return sb.toString(); + } + + private String generateRandomDescription() { + int wordCount = random.nextInt(8) + 3; // 3-10 words + StringBuilder sb = new StringBuilder(); + + for (int i = 0; i < wordCount; i++) { + if (i > 0) { + sb.append(" "); + } + sb.append(SAMPLE_WORDS[random.nextInt(SAMPLE_WORDS.length)]); + } + + return sb.toString(); + } + + /** Random date in the five years before {@link #DATE_ANCHOR}. */ + private LocalDate generateRandomDate() { + LocalDate fiveYearsAgo = DATE_ANCHOR.minusYears(5); + long daysBetween = ChronoUnit.DAYS.between(fiveYearsAgo, DATE_ANCHOR); + long randomDays = Math.floorMod(random.nextLong(), daysBetween); + return fiveYearsAgo.plusDays(randomDays); + } + + /** Random datetime in the year before {@link #DATE_ANCHOR}. */ + private LocalDateTime generateRandomDateTime() { + LocalDateTime oneYearAgo = DATE_ANCHOR.atStartOfDay().minusYears(1); + long secondsBetween = ChronoUnit.SECONDS.between(oneYearAgo, DATE_ANCHOR.atStartOfDay()); + long randomSeconds = Math.floorMod(random.nextLong(), secondsBetween); + return oneYearAgo.plusSeconds(randomSeconds); + } + + private static final DataGenerator defaultGenerator = new DataGenerator(42L); + + /** + * Generate test data with the default generator (fixed seed). + */ + public static List generateTestData(BenchmarkConfiguration.DatasetSize size) { + return defaultGenerator.generateData(size); + } +} diff --git a/fesod-benchmark/src/test/java/org/apache/fesod/sheet/benchmark/util/BenchmarkFileUtilTest.java b/fesod-benchmark/src/test/java/org/apache/fesod/sheet/benchmark/util/BenchmarkFileUtilTest.java new file mode 100644 index 000000000..8d35cb8c6 --- /dev/null +++ b/fesod-benchmark/src/test/java/org/apache/fesod/sheet/benchmark/util/BenchmarkFileUtilTest.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.util; + +import java.io.File; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class BenchmarkFileUtilTest { + + @Test + void createTestFilePlacesFileInTestDataDirectory() { + File file = BenchmarkFileUtil.createTestFile("unit_test_sample.csv"); + + Assertions.assertEquals("unit_test_sample.csv", file.getName()); + Assertions.assertTrue(file.getParentFile().exists()); + Assertions.assertTrue(file.getPath() + .replace(File.separatorChar, '/') + .endsWith("target/benchmark-testdata/unit_test_sample.csv")); + } + + @Test + void createTestDataDirectoryIsIdempotent() { + BenchmarkFileUtil.createTestDataDirectory(); + BenchmarkFileUtil.createTestDataDirectory(); + + File dir = BenchmarkFileUtil.createTestFile("unit_test_dir_check.tmp").getParentFile(); + Assertions.assertTrue(dir.isDirectory()); + } +} diff --git a/fesod-benchmark/src/test/java/org/apache/fesod/sheet/benchmark/util/DataGeneratorTest.java b/fesod-benchmark/src/test/java/org/apache/fesod/sheet/benchmark/util/DataGeneratorTest.java new file mode 100644 index 000000000..e9394a09f --- /dev/null +++ b/fesod-benchmark/src/test/java/org/apache/fesod/sheet/benchmark/util/DataGeneratorTest.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.util; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.fesod.sheet.benchmark.BenchmarkConfiguration; +import org.apache.fesod.sheet.benchmark.data.BenchmarkData; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class DataGeneratorTest { + + private static final int ROWS = 200; + + @Test + void sameSeedProducesIdenticalData() { + List first = new DataGenerator(42L).generateData(ROWS); + List second = new DataGenerator(42L).generateData(ROWS); + + Assertions.assertEquals( + first.stream().map(BenchmarkData::toString).collect(Collectors.toList()), + second.stream().map(BenchmarkData::toString).collect(Collectors.toList())); + } + + @Test + void defaultConstructorUsesFixedSeed() { + List fromDefault = new DataGenerator().generateData(ROWS); + List fromSeed42 = new DataGenerator(42L).generateData(ROWS); + + Assertions.assertEquals( + fromSeed42.stream().map(BenchmarkData::toString).collect(Collectors.toList()), + fromDefault.stream().map(BenchmarkData::toString).collect(Collectors.toList())); + } + + @Test + void differentSeedsProduceDifferentData() { + List first = new DataGenerator(42L) + .generateData(ROWS).stream().map(BenchmarkData::toString).collect(Collectors.toList()); + List second = new DataGenerator(43L) + .generateData(ROWS).stream().map(BenchmarkData::toString).collect(Collectors.toList()); + + Assertions.assertFalse(first.equals(second)); + } + + @Test + void generatedRowsAreComplete() { + List data = new DataGenerator(7L).generateData(ROWS); + + Assertions.assertEquals(ROWS, data.size()); + for (int i = 0; i < data.size(); i++) { + BenchmarkData row = data.get(i); + Assertions.assertEquals(i + 1L, row.getId()); + Assertions.assertFalse(row.getStringData().isEmpty()); + Assertions.assertFalse(row.getCategory().isEmpty()); + Assertions.assertFalse(row.getStatus().isEmpty()); + } + } + + @Test + void datesStayWithinAnchoredWindows() { + List data = new DataGenerator(7L).generateData(ROWS); + + LocalDate dateLowerBound = LocalDate.of(2019, 1, 1); + LocalDate dateUpperBound = LocalDate.of(2024, 1, 1); + LocalDateTime dateTimeLowerBound = LocalDateTime.of(2023, 1, 1, 0, 0); + LocalDateTime dateTimeUpperBound = LocalDateTime.of(2024, 1, 1, 0, 0); + + for (BenchmarkData row : data) { + Assertions.assertTrue(row.getDateValue().isAfter(dateLowerBound.minusDays(1))); + Assertions.assertTrue(row.getDateValue().isBefore(dateUpperBound.plusDays(1))); + Assertions.assertFalse(row.getDateTimeValue().isBefore(dateTimeLowerBound)); + Assertions.assertFalse(row.getDateTimeValue().isAfter(dateTimeUpperBound)); + } + } + + @Test + void generateTestDataMatchesDatasetSize() { + for (BenchmarkConfiguration.DatasetSize size : BenchmarkConfiguration.DatasetSize.values()) { + Assertions.assertEquals( + size.getRowCount(), DataGenerator.generateTestData(size).size()); + } + } +} diff --git a/pom.xml b/pom.xml index 06c6b06d8..a91c9dd46 100644 --- a/pom.xml +++ b/pom.xml @@ -61,6 +61,7 @@ under the License. fesod-common fesod-shaded fesod-sheet + fesod-benchmark From b88ae467f017ed1afa24e90d3b4a6dc3c142299a Mon Sep 17 00:00:00 2001 From: gongzhongqiang Date: Mon, 12 Oct 2026 18:20:29 +0800 Subject: [PATCH 2/5] refactor(benchmark): use Lombok for the benchmark data model Replace the hand-written accessors in BenchmarkData with @Data and @NoArgsConstructor (283 -> 92 lines), per review feedback. Lombok is injected globally by the root pom (provided scope); the module's annotationProcessorPaths overrides the root compiler config for the JMH processor, so Lombok is re-added there alongside it. End-to-end verified: full build and test suite pass, and the read/write/streaming benchmarks produce the same numbers as with the hand-written accessors (Lombok emits identical bytecode). --- fesod-benchmark/pom.xml | 5 + .../sheet/benchmark/data/BenchmarkData.java | 193 +----------------- 2 files changed, 9 insertions(+), 189 deletions(-) diff --git a/fesod-benchmark/pom.xml b/fesod-benchmark/pom.xml index 6ddb3f66a..2fdf2b876 100644 --- a/fesod-benchmark/pom.xml +++ b/fesod-benchmark/pom.xml @@ -97,6 +97,11 @@ under the License. ${maven.compiler.source} ${maven.compiler.target} + + org.projectlombok + lombok + ${lombok.version} + org.openjdk.jmh jmh-generator-annprocess diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/data/BenchmarkData.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/data/BenchmarkData.java index 174339ee4..a81eb5342 100644 --- a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/data/BenchmarkData.java +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/data/BenchmarkData.java @@ -22,12 +22,16 @@ import java.math.BigDecimal; import java.time.LocalDate; import java.time.LocalDateTime; +import lombok.Data; +import lombok.NoArgsConstructor; import org.apache.fesod.sheet.annotation.ExcelProperty; import org.apache.fesod.sheet.annotation.format.DateTimeFormat; /** * Standard benchmark data model with various data types for comprehensive testing */ +@Data +@NoArgsConstructor public class BenchmarkData { @ExcelProperty(value = "ID", index = 0) @@ -91,193 +95,4 @@ public class BenchmarkData { @ExcelProperty(value = "Extra Data 5", index = 19) private String extraData5; - - // Default constructor - public BenchmarkData() {} - - // Getters and setters - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public String getStringData() { - return stringData; - } - - public void setStringData(String stringData) { - this.stringData = stringData; - } - - public Integer getIntValue() { - return intValue; - } - - public void setIntValue(Integer intValue) { - this.intValue = intValue; - } - - public Long getLongValue() { - return longValue; - } - - public void setLongValue(Long longValue) { - this.longValue = longValue; - } - - public Double getDoubleValue() { - return doubleValue; - } - - public void setDoubleValue(Double doubleValue) { - this.doubleValue = doubleValue; - } - - public BigDecimal getBigDecimalValue() { - return bigDecimalValue; - } - - public void setBigDecimalValue(BigDecimal bigDecimalValue) { - this.bigDecimalValue = bigDecimalValue; - } - - public Boolean getBooleanFlag() { - return booleanFlag; - } - - public void setBooleanFlag(Boolean booleanFlag) { - this.booleanFlag = booleanFlag; - } - - public LocalDate getDateValue() { - return dateValue; - } - - public void setDateValue(LocalDate dateValue) { - this.dateValue = dateValue; - } - - public LocalDateTime getDateTimeValue() { - return dateTimeValue; - } - - public void setDateTimeValue(LocalDateTime dateTimeValue) { - this.dateTimeValue = dateTimeValue; - } - - public String getCategory() { - return category; - } - - public void setCategory(String category) { - this.category = category; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getStatus() { - return status; - } - - public void setStatus(String status) { - this.status = status; - } - - public Float getFloatValue() { - return floatValue; - } - - public void setFloatValue(Float floatValue) { - this.floatValue = floatValue; - } - - public Short getShortValue() { - return shortValue; - } - - public void setShortValue(Short shortValue) { - this.shortValue = shortValue; - } - - public Byte getByteValue() { - return byteValue; - } - - public void setByteValue(Byte byteValue) { - this.byteValue = byteValue; - } - - public String getExtraData1() { - return extraData1; - } - - public void setExtraData1(String extraData1) { - this.extraData1 = extraData1; - } - - public String getExtraData2() { - return extraData2; - } - - public void setExtraData2(String extraData2) { - this.extraData2 = extraData2; - } - - public String getExtraData3() { - return extraData3; - } - - public void setExtraData3(String extraData3) { - this.extraData3 = extraData3; - } - - public String getExtraData4() { - return extraData4; - } - - public void setExtraData4(String extraData4) { - this.extraData4 = extraData4; - } - - public String getExtraData5() { - return extraData5; - } - - public void setExtraData5(String extraData5) { - this.extraData5 = extraData5; - } - - @Override - public String toString() { - return "BenchmarkData{" + "id=" - + id + ", stringData='" - + stringData + '\'' + ", intValue=" - + intValue + ", longValue=" - + longValue + ", doubleValue=" - + doubleValue + ", bigDecimalValue=" - + bigDecimalValue + ", booleanFlag=" - + booleanFlag + ", dateValue=" - + dateValue + ", dateTimeValue=" - + dateTimeValue + ", category='" - + category + '\'' + ", description='" - + description + '\'' + ", status='" - + status + '\'' + ", floatValue=" - + floatValue + ", shortValue=" - + shortValue + ", byteValue=" - + byteValue + ", extraData1='" - + extraData1 + '\'' + ", extraData2='" - + extraData2 + '\'' + ", extraData3='" - + extraData3 + '\'' + ", extraData4='" - + extraData4 + '\'' + ", extraData5='" - + extraData5 + '\'' + '}'; - } } From 2ad00b3a7396124ed83c2323d0f9809af7e0388d Mon Sep 17 00:00:00 2001 From: gongzhongqiang Date: Mon, 12 Oct 2026 18:26:11 +0800 Subject: [PATCH 3/5] refactor(benchmark): adopt Lombok across the remaining support classes Module-wide pass over the benchmark sources for Lombok-replaceable boilerplate: - DataGenerator and BenchmarkFileUtil: hand-written SLF4J loggers replaced with @Slf4j (field renamed to the conventional log) - BenchmarkConfiguration: enum accessors replaced with @Getter The JMH suite classes and tests have no replaceable boilerplate (no loggers, no accessors; @Param fields are JMH-managed). --- .../sheet/benchmark/BenchmarkConfiguration.java | 16 ++++------------ .../sheet/benchmark/util/BenchmarkFileUtil.java | 8 +++----- .../sheet/benchmark/util/DataGenerator.java | 10 ++++------ 3 files changed, 11 insertions(+), 23 deletions(-) diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/BenchmarkConfiguration.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/BenchmarkConfiguration.java index 8c77524e1..084ac0a9d 100644 --- a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/BenchmarkConfiguration.java +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/BenchmarkConfiguration.java @@ -19,6 +19,8 @@ package org.apache.fesod.sheet.benchmark; +import lombok.Getter; + /** * Shared constants for the benchmark suites. */ @@ -30,6 +32,7 @@ private BenchmarkConfiguration() {} * Dataset sizes used by the benchmark suites. Rows have 20 columns each * (see {@link org.apache.fesod.sheet.benchmark.data.BenchmarkData}). */ + @Getter public enum DatasetSize { SMALL(1_000, "1K"), MEDIUM(10_000, "10K"), @@ -42,20 +45,13 @@ public enum DatasetSize { this.rowCount = rowCount; this.label = label; } - - public int getRowCount() { - return rowCount; - } - - public String getLabel() { - return label; - } } /** * File formats covered by the benchmark suites. Note that XLS is limited * to 65,536 rows per sheet. */ + @Getter public enum FileFormat { XLSX("xlsx"), XLS("xls"), @@ -66,9 +62,5 @@ public enum FileFormat { FileFormat(String extension) { this.extension = extension; } - - public String getExtension() { - return extension; - } } } diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/util/BenchmarkFileUtil.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/util/BenchmarkFileUtil.java index cd3a09f4b..0d44bb702 100644 --- a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/util/BenchmarkFileUtil.java +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/util/BenchmarkFileUtil.java @@ -24,18 +24,16 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; /** * Central place for the scratch directory benchmark files are written to * ({@code target/benchmark-testdata}), so all suites share one location that * Maven cleans up on {@code mvn clean}. */ +@Slf4j public class BenchmarkFileUtil { - private static final Logger logger = LoggerFactory.getLogger(BenchmarkFileUtil.class); - private static final String TEST_DATA_DIR = "target/benchmark-testdata"; /** @@ -71,7 +69,7 @@ public static void delete(File file) { try { Files.deleteIfExists(file.toPath()); } catch (IOException e) { - logger.warn("Failed to delete benchmark file {}: {}", file, e.getMessage()); + log.warn("Failed to delete benchmark file {}: {}", file, e.getMessage()); } } } diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/util/DataGenerator.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/util/DataGenerator.java index 9cab6ed5e..e5fb2daca 100644 --- a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/util/DataGenerator.java +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/util/DataGenerator.java @@ -27,20 +27,18 @@ import java.util.ArrayList; import java.util.List; import java.util.Random; +import lombok.extern.slf4j.Slf4j; import org.apache.fesod.sheet.benchmark.BenchmarkConfiguration; import org.apache.fesod.sheet.benchmark.data.BenchmarkData; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Generates deterministic benchmark data: a fixed default seed and a fixed date * anchor mean the same seed always produces the same rows, so benchmark results * from different runs are comparable. */ +@Slf4j public class DataGenerator { - private static final Logger logger = LoggerFactory.getLogger(DataGenerator.class); - /** Fixed anchor for generated dates; never uses the wall clock. */ private static final LocalDate DATE_ANCHOR = LocalDate.of(2024, 1, 1); @@ -112,7 +110,7 @@ public List generateData(BenchmarkConfiguration.DatasetSize size) } public List generateData(int rowCount) { - logger.info("Generating {} rows of benchmark data", rowCount); + log.info("Generating {} rows of benchmark data", rowCount); List data = new ArrayList<>(rowCount); long startTime = System.currentTimeMillis(); @@ -122,7 +120,7 @@ public List generateData(int rowCount) { } long duration = System.currentTimeMillis() - startTime; - logger.info( + log.info( "Generated {} rows in {} ms ({} rows/sec)", rowCount, duration, From 425d3707d0e399a29c3c1fa9b074e23d7e0d6895 Mon Sep 17 00:00:00 2001 From: gongzhongqiang Date: Mon, 12 Oct 2026 19:47:56 +0800 Subject: [PATCH 4/5] refactor(benchmark): use 3 forks for the comparison suites too Align the fork count of FesodVsPoiBenchmark and StreamingBenchmark with FesodBenchmark so every suite exposes cross-fork JVM/JIT variability in its error bars and the suites are comparable with each other. The original single fork on the comparison suites was only a wall-clock convenience for manual runs, not a measurement choice. Also document the deliberate heap difference (core suite 1g/G1GC vs comparison 2g) under the contracts table. --- fesod-benchmark/benchmark.md | 10 ++++++++-- .../fesod/sheet/benchmark/FesodVsPoiBenchmark.java | 2 +- .../fesod/sheet/benchmark/StreamingBenchmark.java | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/fesod-benchmark/benchmark.md b/fesod-benchmark/benchmark.md index fc3b476fa..3672e8ab2 100644 --- a/fesod-benchmark/benchmark.md +++ b/fesod-benchmark/benchmark.md @@ -59,8 +59,14 @@ results collected at different times stay comparable: | Suite | Mode | Forks | Warmup | Measurement | JVM args | |---|---|---|---|---|---| | `FesodBenchmark` | avgt, ms/op | 3 | 3 × 1s | 5 × 2s | `-Xms1g -Xmx1g -XX:+UseG1GC` | -| `FesodVsPoiBenchmark` | avgt, ms/op | 1 | 3 × 5s | 5 × 5s | `-Xms2g -Xmx2g` | -| `StreamingBenchmark` | avgt, ms/op | 1 | 3 × 5s | 5 × 5s | `-Xms2g -Xmx2g` | +| `FesodVsPoiBenchmark` | avgt, ms/op | 3 | 3 × 5s | 5 × 5s | `-Xms2g -Xmx2g` | +| `StreamingBenchmark` | avgt, ms/op | 3 | 3 × 5s | 5 × 5s | `-Xms2g -Xmx2g` | + +All suites use 3 forks so cross-fork JVM/JIT variability is visible in the +error bars. The core suite deliberately keeps a smaller fixed heap (`1g`, +G1GC) — a roomy constant heap removes GC adaptivity from the timing signal — +while the comparison suites use `2g` so POI's in-memory workloads are not the +constraint being measured. ## Notes on interpreting results diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/FesodVsPoiBenchmark.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/FesodVsPoiBenchmark.java index 7e396504c..87aa15050 100644 --- a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/FesodVsPoiBenchmark.java +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/FesodVsPoiBenchmark.java @@ -93,7 +93,7 @@ @Warmup(iterations = 3, time = 5, timeUnit = TimeUnit.SECONDS) @Measurement(iterations = 5, time = 5, timeUnit = TimeUnit.SECONDS) @Fork( - value = 1, + value = 3, jvmArgs = {"-Xms2g", "-Xmx2g"}) public class FesodVsPoiBenchmark { diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/StreamingBenchmark.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/StreamingBenchmark.java index 9b45975b6..c2466ddbc 100644 --- a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/StreamingBenchmark.java +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/StreamingBenchmark.java @@ -93,7 +93,7 @@ @Warmup(iterations = 3, time = 5, timeUnit = TimeUnit.SECONDS) @Measurement(iterations = 5, time = 5, timeUnit = TimeUnit.SECONDS) @Fork( - value = 1, + value = 3, jvmArgs = {"-Xms2g", "-Xmx2g"}) public class StreamingBenchmark { From d60c05b32f50e00eb2f570e36f73c0807d6dbc2c Mon Sep 17 00:00:00 2001 From: gongzhongqiang Date: Tue, 6 Oct 2026 18:41:25 +0800 Subject: [PATCH 5/5] feat(benchmark): propose release-tag performance regression gate Wire FesodBenchmark into a performance regression gate (split out of - BaselineRunner pins the execution contract (3 forks, 3x1s warmup, 5x2s measurement, fixed JVM args, gc profiler) and writes JMH JSON - BaselineComparator compares against the committed baseline with tiered, noise-aware verdicts (alloc/op hard-fail; time regressions gated on JMH error-bar overlap) and renders the Markdown report - benchmark.yml runs the gate on release tags and manual dispatch only, bootstraps the baseline on first run via an automated PR, and posts regression reports on the matching GitHub Release (best-effort notification ladder) No baseline is committed up front - the first run on an apache runner bootstraps it through an automated PR, so the committed baseline has apache-runner provenance. --- .github/workflows/benchmark.yml | 363 +++++++++ fesod-benchmark/baseline/README.md | 15 + fesod-benchmark/benchmark.md | 42 +- fesod-benchmark/pom.xml | 6 + .../baseline/BaselineComparator.java | 712 ++++++++++++++++++ .../benchmark/baseline/BaselineRunner.java | 155 ++++ 6 files changed, 1291 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/benchmark.yml create mode 100644 fesod-benchmark/baseline/README.md create mode 100644 fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/baseline/BaselineComparator.java create mode 100644 fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/baseline/BaselineRunner.java diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000..b77956b56 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,363 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Performance regression gate for Fesod. +# +# Runs the baseline benchmark suite (fesod-benchmark/baseline package) on release tags +# (e.g. 2.1.0-incubating) and on demand (workflow_dispatch), compares the results against +# the committed baseline (fesod-benchmark/baseline/jmh-baseline.json) and: +# - appends the comparison report to the job step summary +# - emits ::error/::warning annotations for regressions (visible on the run page) +# - fails the job when a regression exceeds the fail threshold with non-overlapping +# JMH error bars (high-confidence regressions only — see benchmark.md for the +# rationale of the tiered gate) +# +# On tags with a regression, a follow-up job posts the report as a comment on the +# matching GitHub Release (or opens an issue) so maintainers cannot miss it. +# A passing tag run (or a dispatch with update_baseline) opens a PR that refreshes +# the committed baseline — the baseline lifecycle follows releases. +# +# The baseline can only be regenerated on a GitHub runner (same hardware/JDK). +# Never commit a baseline generated on a local machine. + +name: Benchmark + +on: + push: + tags: + - '[0-9]+.*' + workflow_dispatch: + inputs: + update_baseline: + description: 'Run the suite and open a PR refreshing the committed baseline' + type: boolean + default: false + warn_threshold: + description: 'Regression percentage that triggers a warning' + type: number + default: 10 + fail_threshold: + description: 'Regression percentage that fails the job (beyond JMH error bars)' + type: number + default: 20 + +concurrency: + group: benchmark-${{ github.ref }} + cancel-in-progress: false + +permissions: + contents: read + +env: + BASELINE_FILE: fesod-benchmark/baseline/jmh-baseline.json + BASELINE_META: fesod-benchmark/baseline/baseline-meta.json + CURRENT_RESULT: fesod-benchmark/target/baseline-current.json + REPORT_FILE: fesod-benchmark/target/benchmark-report.md + +jobs: + benchmark: + name: Baseline comparison (JDK 17) + runs-on: ubuntu-24.04 + timeout-minutes: 45 + permissions: + contents: read + pull-requests: write + outputs: + bootstrap: ${{ steps.baseline.outputs.exists != 'true' }} + update_requested: ${{ github.event_name == 'workflow_dispatch' && inputs.update_baseline }} + is_tag: ${{ startsWith(github.ref, 'refs/tags/') }} + compare_outcome: ${{ steps.compare.outcome }} + tag: ${{ github.ref_name }} + jdk: ${{ steps.env.outputs.jdk }} + os: ${{ steps.env.outputs.os }} + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Set up JDK 17 + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: '17' + + - name: Cache local Maven repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-benchmark-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven-benchmark- + + - name: Record environment + id: env + run: | + echo "jdk=$(java -version 2>&1 | head -n 1)" >> "$GITHUB_OUTPUT" + echo "os=$(uname -sr)" >> "$GITHUB_OUTPUT" + + - name: Check baseline presence + id: baseline + run: | + if [ -f "$BASELINE_FILE" ]; then + echo "exists=true" >> "$GITHUB_OUTPUT" + else + echo "exists=false" >> "$GITHUB_OUTPUT" + echo "::notice::No committed baseline found — this run will bootstrap one." + fi + + - name: Build benchmark suite + run: ./mvnw -B -ntp -pl fesod-benchmark -am package -DskipTests + + - name: Run baseline suite + run: | + cd fesod-benchmark + java -cp target/benchmarks.jar org.apache.fesod.sheet.benchmark.baseline.BaselineRunner + + - name: Compare against baseline + id: compare + continue-on-error: true + run: | + java -cp fesod-benchmark/target/benchmarks.jar \ + org.apache.fesod.sheet.benchmark.baseline.BaselineComparator \ + --baseline "$BASELINE_FILE" \ + --baseline-meta "$BASELINE_META" \ + --current "$CURRENT_RESULT" \ + --warn "${{ inputs.warn_threshold || vars.BENCHMARK_WARN_PCT || '10' }}" \ + --fail "${{ inputs.fail_threshold || vars.BENCHMARK_FAIL_PCT || '20' }}" \ + --report "$REPORT_FILE" + + - name: Add report to job summary + if: always() + run: | + RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + { + echo "## 📊 Fesod performance report" + echo + echo "Run \`${{ github.sha }}\` on \`ubuntu-24.04\` · ${{ steps.env.outputs.jdk }} · [full run](${RUN_URL})" + echo + if [ -f "$REPORT_FILE" ]; then + cat "$REPORT_FILE" + else + echo "_No comparison report was produced (suite run failed or was cancelled)." + fi + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload benchmark results + if: always() + uses: actions/upload-artifact@v4 + with: + name: benchmark-results + if-no-files-found: warn + path: | + ${{ env.CURRENT_RESULT }} + ${{ env.REPORT_FILE }} + + - name: Comment report on open PR (manual runs on a PR branch) + if: github.event_name == 'workflow_dispatch' && always() + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + BRANCH: ${{ github.ref_name }} + run: | + [ -f "$REPORT_FILE" ] || { echo "No report file — skipping."; exit 0; } + PR_NUMBER=$(gh pr list --repo "$GITHUB_REPOSITORY" --head "$BRANCH" --state open \ + --json number -q '.[0].number' 2>/dev/null || true) + if [ -z "$PR_NUMBER" ]; then + echo "No open PR for branch '$BRANCH' — report is in the job summary only." + exit 0 + fi + MARKER='' + RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + BODY_FILE=$(mktemp) + { + echo "$MARKER" + echo "### 📊 Performance vs baseline — run [${{ github.run_id }}](${RUN_URL})" + echo + cat "$REPORT_FILE" + } > "$BODY_FILE" + EXISTING=$(gh api "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \ + --paginate -q ".[] | select(.body | startswith(\"$MARKER\")) | .id" | head -n 1) + if [ -n "$EXISTING" ]; then + gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$EXISTING" \ + -F body=@"$BODY_FILE" > /dev/null + else + gh api -X POST "repos/$GITHUB_REPOSITORY/issues/$PR_NUMBER/comments" \ + -F body=@"$BODY_FILE" > /dev/null + fi + echo "Report commented on PR #$PR_NUMBER." + + - name: Fail on regression + if: steps.compare.outcome == 'failure' && !(github.event_name == 'workflow_dispatch' && inputs.update_baseline) + run: | + echo "::error title=Performance regression::At least one benchmark regressed beyond the fail threshold. See the benchmark report in the job summary." + exit 1 + + notify-regression: + name: Notify maintainers (tag regression) + needs: benchmark + if: >- + always() && + !cancelled() && + needs.benchmark.outputs.compare_outcome == 'failure' && + needs.benchmark.outputs.is_tag == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + contents: write + issues: write + steps: + - name: Download benchmark results + uses: actions/download-artifact@v4 + with: + name: benchmark-results + path: fesod-benchmark/target/ + + - name: Post report on release (or open an issue) + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ needs.benchmark.outputs.tag }} + run: | + if [ ! -f "$REPORT_FILE" ]; then + echo "No report file — nothing to notify." + exit 0 + fi + BODY_FILE=$(mktemp) + { + echo "## 🚨 Performance regression detected on tag \`${TAG}\`" + echo + echo "The [benchmark run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) found regressions beyond the fail threshold. The baseline was **not** advanced." + echo + cat "$REPORT_FILE" + } > "$BODY_FILE" + + # Best-effort notification ladder: release comment -> issue -> warning + # annotation. A missing release or a repository with issues disabled must + # never turn this job red — the gate itself is already red in that case. + notified=1 + RELEASE_ID=$(gh api "repos/$GITHUB_REPOSITORY/releases/tags/$TAG" -q .id 2>/dev/null || true) + # a failed lookup can leak the error body into the variable — keep numeric ids only + case "$RELEASE_ID" in + ''|*[!0-9]*) RELEASE_ID='' ;; + esac + if [ -n "$RELEASE_ID" ]; then + if gh api -X POST "repos/$GITHUB_REPOSITORY/releases/$RELEASE_ID/comments" \ + -F body=@"$BODY_FILE" > /dev/null 2>&1; then + echo "Commented on release $TAG." + notified=0 + else + echo "::warning title=Regression notification::Release comment failed for '$TAG' — falling back to an issue." + fi + fi + if [ "$notified" -ne 0 ]; then + if gh issue create --repo "$GITHUB_REPOSITORY" \ + --title "🚨 Performance regression on tag ${TAG}" \ + --body-file "$BODY_FILE" --label "performance" 2>/dev/null \ + || gh issue create --repo "$GITHUB_REPOSITORY" \ + --title "🚨 Performance regression on tag ${TAG}" \ + --body-file "$BODY_FILE"; then + echo "Opened a tracking issue." + notified=0 + fi + fi + if [ "$notified" -ne 0 ]; then + echo "::warning title=Regression notification::Could not comment on release '$TAG' or open an issue (no release / issues disabled?). The report is in the job summary and run annotations." + fi + + update-baseline: + name: Refresh baseline (opens PR) + needs: benchmark + if: >- + !cancelled() && + needs.benchmark.result == 'success' && + (needs.benchmark.outputs.update_requested == 'true' || + needs.benchmark.outputs.bootstrap == 'true' || + (needs.benchmark.outputs.is_tag == 'true' && needs.benchmark.outputs.compare_outcome == 'success')) + runs-on: ubuntu-24.04 + timeout-minutes: 15 + permissions: + contents: write + pull-requests: write + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Download benchmark results + uses: actions/download-artifact@v4 + with: + name: benchmark-results + path: fesod-benchmark/target/ + + - name: Write baseline files + env: + JDK_INFO: ${{ needs.benchmark.outputs.jdk }} + OS_INFO: ${{ needs.benchmark.outputs.os }} + SOURCE_REF: ${{ needs.benchmark.outputs.is_tag == 'true' && needs.benchmark.outputs.tag || github.ref_name }} + run: | + mkdir -p fesod-benchmark/baseline + cp "$CURRENT_RESULT" "$BASELINE_FILE" + jq -n \ + --arg generatedAt "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --arg gitSha "${{ github.sha }}" \ + --arg gitRef "$SOURCE_REF" \ + --arg jdkVersion "$JDK_INFO" \ + --arg osName "$OS_INFO" \ + --arg runnerLabel "ubuntu-24.04" \ + --argjson benchmarkCount "$(jq 'length' "$CURRENT_RESULT")" \ + '{generatedAt: $generatedAt, gitSha: $gitSha, gitRef: $gitRef, jdkVersion: $jdkVersion, + osName: $osName, runnerLabel: $runnerLabel, benchmarkCount: $benchmarkCount}' \ + > "$BASELINE_META" + cat "$BASELINE_META" + + - name: Open baseline refresh PR + env: + GH_TOKEN: ${{ github.token }} + BASE_BRANCH: ${{ needs.benchmark.outputs.is_tag == 'true' && 'main' || github.ref_name }} + REASON: ${{ needs.benchmark.outputs.is_tag == 'true' && format('release tag {0}', needs.benchmark.outputs.tag) || github.ref_name }} + run: | + BRANCH="benchmark/baseline-$(date -u +%Y%m%d%H%M%S)" + PR_BODY_FILE=$(mktemp) + { + echo "## Performance baseline refresh" + echo + echo "Regenerated from \`${{ github.sha }}\` (\`$REASON\`) on a \`ubuntu-24.04\` runner with JDK 17 (Temurin)" + echo "by the [Benchmark workflow](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})." + echo + if [ -f "$REPORT_FILE" ]; then + echo "### Change vs previous baseline" + echo + cat "$REPORT_FILE" + echo + fi + echo "### Files changed" + echo + echo '- `fesod-benchmark/baseline/jmh-baseline.json` — fresh JMH results, the new reference' + echo '- `fesod-benchmark/baseline/baseline-meta.json` — provenance (commit, runner, JDK, date)' + echo + echo "Merge this PR to accept the new performance characteristics as the project baseline." + } > "$PR_BODY_FILE" + + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + git checkout -b "$BRANCH" + git add fesod-benchmark/baseline + git commit -m "chore(benchmark): refresh performance baseline from ${{ github.sha }}" + git push origin "$BRANCH" + + gh pr create \ + --base "$BASE_BRANCH" \ + --head "$BRANCH" \ + --title "chore(benchmark): refresh performance baseline" \ + --body-file "$PR_BODY_FILE" diff --git a/fesod-benchmark/baseline/README.md b/fesod-benchmark/baseline/README.md new file mode 100644 index 000000000..a7c03ea68 --- /dev/null +++ b/fesod-benchmark/baseline/README.md @@ -0,0 +1,15 @@ +# Committed performance baseline + +This directory holds the reference results for the performance regression gate +(see [`../benchmark.md`](../benchmark.md)): + +- `jmh-baseline.json` — JMH results of the last accepted `FesodBenchmark` run. Generated by the + *Benchmark* workflow; **must only ever be produced on an `ubuntu-24.04` GitHub runner with JDK 17 + (Temurin)**, because comparisons are absolute. Never commit a baseline generated on a local machine. +- `baseline-meta.json` — provenance of the baseline: source commit, JDK, runner, generation date. + +On the first run after this branch merges, no baseline exists yet: the workflow detects the +bootstrap case, treats every benchmark as NEW (informational, gate passes) and opens a PR that +commits the initial baseline. After that, refresh it via +*Actions → Benchmark → Run workflow* with **update_baseline** checked; the workflow opens +a PR containing the new baseline plus a delta report against the previous one. diff --git a/fesod-benchmark/benchmark.md b/fesod-benchmark/benchmark.md index 3672e8ab2..4bd2986d2 100644 --- a/fesod-benchmark/benchmark.md +++ b/fesod-benchmark/benchmark.md @@ -1,7 +1,8 @@ # Fesod Benchmark Module -JMH benchmarks for Fesod spreadsheet operations. This module is a **manual tool**: -it is not wired into any CI workflow and is not part of the Fesod public API. +JMH benchmarks for Fesod spreadsheet operations, plus (per this proposal) a +release-time performance regression gate. Benchmark code is not part of the +Fesod public API. It covers three complementary suites (package `org.apache.fesod.sheet.benchmark`): @@ -107,3 +108,40 @@ constraint being measured. (overwritten each invocation) and delete it from the trial `@TearDown` via `BenchmarkFileUtil.delete` — never inside the measured method, so cleanup I/O stays out of the reported time. + +## Performance regression gate (this PR's proposal) + +On top of the manual suites above, this PR wires `FesodBenchmark` into a +**release-time regression gate**: + +- **Triggers**: release tags (`[0-9]+.*`) and manual `workflow_dispatch` — never + per pull request (shared-runner noise makes every-push gating flaky and + wasteful). Workflow: [`.github/workflows/benchmark.yml`](../.github/workflows/benchmark.yml). +- **Components** (package `org.apache.fesod.sheet.benchmark.baseline`): + `BaselineRunner` runs `FesodBenchmark` with the pinned contract (3 forks, + 3×1s warmup, 5×2s measurement, `-Xms1g -Xmx1g -XX:+UseG1GC`, gc profiler); + `BaselineComparator` compares against the committed baseline + ([`baseline/`](baseline/)) and renders the Markdown report with the gate verdict. +- **Tiered verdicts** (calibrated against measured noise on real runners): + + | Signal | Noise | Decision | + |---|---|---| + | `gc.alloc.rate.norm` (alloc per op) | ±0.1% | Regression beyond threshold → **fail** | + | avgt beyond fail threshold **and** non-overlapping JMH error bars | ±7% typical | **Fail** | + | avgt beyond threshold with overlapping error bars | — | **WARN only** | + | Tracked benchmark missing from a run | — | **Fail** | + + Thresholds default to warn 10% / fail 20% (overridable via dispatch inputs or + repo variables `BENCHMARK_WARN_PCT` / `BENCHMARK_FAIL_PCT`). +- **Baseline lifecycle**: the baseline is only ever generated on + `ubuntu-24.04` + JDK 17 Temurin GitHub runners; the first run bootstraps it + via an automated PR, passing tags advance it via automated PRs, and a + regression on a tag posts the report on the matching GitHub Release (or + opens an issue). Benchmark method/param names are the baseline keys — do not + rename them without a baseline refresh. +- **ASF compliance**: only `actions/*` are used; bot pushes go only to + `benchmark/baseline-*` branches; baseline JSON enters the repo exclusively + through reviewed PRs. + +This gate is the part of the original PR #575 that was split out per review +feedback; it is proposed here separately for discussion. diff --git a/fesod-benchmark/pom.xml b/fesod-benchmark/pom.xml index 2fdf2b876..55bbbde56 100644 --- a/fesod-benchmark/pom.xml +++ b/fesod-benchmark/pom.xml @@ -80,6 +80,12 @@ under the License. slf4j-simple + + + com.alibaba.fastjson2 + fastjson2 + + org.junit.jupiter diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/baseline/BaselineComparator.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/baseline/BaselineComparator.java new file mode 100644 index 000000000..9f8baeaaf --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/baseline/BaselineComparator.java @@ -0,0 +1,712 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.baseline; + +import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONArray; +import com.alibaba.fastjson2.JSONObject; +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.TreeMap; +import java.util.TreeSet; + +/** + * Compares a fresh {@link BaselineRunner} result file against the committed baseline and + * renders a Markdown report for humans (PR comment / job summary), plus a process exit + * code used as the CI gate. + * + *

Comparison rules: + *

+ * + *

Exit codes: {@code 0} = pass (or bootstrap mode without baseline), + * {@code 1} = regression gate failed, {@code 2} = usage or I/O error. + */ +public final class BaselineComparator { + + private static final String GC_ALLOC_RATE_NORM = "gc.alloc.rate.norm"; + + private static final String DEFAULT_BASELINE = "fesod-benchmark/baseline/jmh-baseline.json"; + private static final String DEFAULT_BASELINE_META = "fesod-benchmark/baseline/baseline-meta.json"; + private static final String DEFAULT_CURRENT = "target/baseline-current.json"; + + private static final String OK = ":white_check_mark: OK"; + private static final String IMPROVED = ":green_circle: FASTER"; + private static final String WARN = ":warning: WARN"; + private static final String FAIL = ":red_circle: FAIL"; + private static final String NEW = ":new: NEW"; + private static final String MISSING = ":black_circle: MISSING"; + + private static final String UP = ":small_red_triangle:"; + private static final String DOWN = ":small_red_triangle_down:"; + + private enum Verdict { + OK(0), + IMPROVED(1), + WARN(2), + FAIL(3), + NEW(0), + MISSING(3); + + private final int severity; + + Verdict(int severity) { + this.severity = severity; + } + } + + /** One JMH result entry: primary metric plus the gc allocation metric when present. */ + private static final class Metric { + final String key; + final String displayName; + final String mode; + final double score; + final double scoreError; + final String scoreUnit; + final Double allocScore; + final double allocError; + final String allocUnit; + + Metric( + String key, + String displayName, + String mode, + double score, + double scoreError, + String scoreUnit, + Double allocScore, + double allocError, + String allocUnit) { + this.key = key; + this.displayName = displayName; + this.mode = mode; + this.score = score; + this.scoreError = scoreError; + this.scoreUnit = scoreUnit; + this.allocScore = allocScore; + this.allocError = allocError; + this.allocUnit = allocUnit; + } + } + + /** One report row: baseline/current metric pair plus the computed verdict. */ + private static final class Row { + final Metric baseline; + final Metric current; + Verdict verdict; + String timeDetail; + String allocDetail; + + Row(Metric baseline, Metric current) { + this.baseline = baseline; + this.current = current; + } + + Metric currentOrBaseline() { + return current != null ? current : baseline; + } + } + + private final double warnPct; + private final double failPct; + private final boolean allowMissing; + + private BaselineComparator(double warnPct, double failPct, boolean allowMissing) { + this.warnPct = warnPct; + this.failPct = failPct; + this.allowMissing = allowMissing; + } + + public static void main(String[] args) { + String baselinePath = DEFAULT_BASELINE; + String baselineMetaPath = DEFAULT_BASELINE_META; + String currentPath = DEFAULT_CURRENT; + String reportPath = null; + double warnPct = 10.0d; + double failPct = 20.0d; + boolean allowMissing = false; + + for (int i = 0; i < args.length; i++) { + String arg = args[i]; + if ("--baseline".equals(arg) && i + 1 < args.length) { + baselinePath = args[++i]; + } else if ("--baseline-meta".equals(arg) && i + 1 < args.length) { + baselineMetaPath = args[++i]; + } else if ("--current".equals(arg) && i + 1 < args.length) { + currentPath = args[++i]; + } else if ("--report".equals(arg) && i + 1 < args.length) { + reportPath = args[++i]; + } else if ("--warn".equals(arg) && i + 1 < args.length) { + warnPct = Double.parseDouble(args[++i]); + } else if ("--fail".equals(arg) && i + 1 < args.length) { + failPct = Double.parseDouble(args[++i]); + } else if ("--allow-missing".equals(arg)) { + allowMissing = true; + } else { + usageAndExit("Unknown or incomplete argument: " + arg); + } + } + if (failPct <= warnPct) { + usageAndExit("--fail threshold must be greater than --warn threshold"); + } + + try { + File currentFile = new File(currentPath); + if (!currentFile.isFile()) { + System.err.println("ERROR: current result file not found: " + currentPath); + System.exit(2); + } + Map current = index(readResults(currentFile)); + String currentContract = readContract(currentFile); + + File baselineFile = new File(baselinePath); + Map baseline = + baselineFile.isFile() ? index(readResults(baselineFile)) : new TreeMap(); + String baselineContract = baselineFile.isFile() ? readContract(baselineFile) : null; + boolean contractDiffers = baselineContract != null && !baselineContract.equals(currentContract); + + BaselineComparator comparator = new BaselineComparator(warnPct, failPct, allowMissing); + List rows = comparator.compare(baseline, current); + String report = comparator.renderReport( + rows, + baseline.isEmpty(), + readMeta(new File(baselineMetaPath)), + baselineContract, + currentContract, + contractDiffers); + + System.out.println(); + System.out.println(report); + + if (reportPath != null) { + File reportFile = new File(reportPath); + if (reportFile.getParentFile() != null + && !reportFile.getParentFile().exists()) { + reportFile.getParentFile().mkdirs(); + } + Files.write(reportFile.toPath(), report.getBytes(StandardCharsets.UTF_8)); + System.out.println("Report written to: " + reportPath); + System.out.println(); + } + + boolean gateFailed = false; + for (Row row : rows) { + if (row.verdict == Verdict.FAIL || (row.verdict == Verdict.MISSING && !allowMissing)) { + gateFailed = true; + System.err.println("REGRESSION: " + row.verdict + " " + row.currentOrBaseline().displayName + + " — time: " + row.timeDetail + ", alloc: " + row.allocDetail); + } + } + printAnnotations(rows); + if (contractDiffers) { + System.err.println("NOTE: execution contract differs from baseline — refresh the baseline " + + "for a strictly valid comparison."); + } + + System.exit(gateFailed ? 1 : 0); + } catch (IOException e) { + System.err.println("ERROR: " + e.getMessage()); + System.exit(2); + } + } + + /** + * Emits GitHub Actions workflow commands so regressions surface as annotations on the + * run page (and any linked PR/release), not only in the log. No-op outside Actions. + */ + private static void printAnnotations(List rows) { + if (!"true".equals(System.getenv("GITHUB_ACTIONS"))) { + return; + } + for (Row row : rows) { + String detail = (row.currentOrBaseline().displayName + " — time: " + row.timeDetail + ", alloc: " + + row.allocDetail) + .replace(UP + " ", "") + .replace(DOWN + " ", ""); + if (row.verdict == Verdict.FAIL || (row.verdict == Verdict.MISSING)) { + System.out.println("::error title=Performance regression::" + detail); + } else if (row.verdict == Verdict.WARN) { + System.out.println("::warning title=Performance warning::" + detail); + } + } + } + + // ------------------------------------------------------------------ + // Comparison + // ------------------------------------------------------------------ + + private List compare(Map baseline, Map current) { + List rows = new ArrayList(); + TreeSet keys = new TreeSet(); + keys.addAll(baseline.keySet()); + keys.addAll(current.keySet()); + + for (String key : keys) { + Metric base = baseline.get(key); + Metric cur = current.get(key); + Row row = new Row(base, cur); + + if (base == null) { + row.verdict = Verdict.NEW; + row.timeDetail = "not in baseline"; + row.allocDetail = "n/a"; + } else if (cur == null) { + row.verdict = Verdict.MISSING; + row.timeDetail = "missing from current run"; + row.allocDetail = "n/a"; + } else { + row.verdict = worst(metricVerdict(base, cur), allocVerdict(base, cur)); + row.timeDetail = timeDetail(base, cur); + row.allocDetail = allocDetail(base, cur); + } + rows.add(row); + } + return rows; + } + + private Verdict metricVerdict(Metric base, Metric cur) { + boolean higherIsWorse = higherIsWorse(cur.mode); + return verdict(base.score, base.scoreError, cur.score, cur.scoreError, higherIsWorse); + } + + private Verdict allocVerdict(Metric base, Metric cur) { + if (base.allocScore == null || cur.allocScore == null || base.allocScore <= 0 || cur.allocScore <= 0) { + return Verdict.OK; + } + return verdict(base.allocScore, base.allocError, cur.allocScore, cur.allocError, true); + } + + private Verdict verdict(double base, double baseErr, double cur, double curErr, boolean higherIsWorse) { + double regressionPct = higherIsWorse ? (cur - base) / base * 100.0d : (base - cur) / base * 100.0d; + boolean intervalsOverlap = + higherIsWorse ? (cur - curErr) <= (base + baseErr) : (cur + curErr) >= (base - baseErr); + + if (regressionPct >= failPct && !intervalsOverlap) { + return Verdict.FAIL; + } + if (regressionPct >= failPct || regressionPct >= warnPct) { + return Verdict.WARN; + } + if (regressionPct <= -warnPct) { + return Verdict.IMPROVED; + } + return Verdict.OK; + } + + private String timeDetail(Metric base, Metric cur) { + return changeDetail(cur.score - base.score, base.score); + } + + private String allocDetail(Metric base, Metric cur) { + if (base.allocScore == null || cur.allocScore == null || base.allocScore <= 0) { + return "n/a"; + } + return formatBytes(cur.allocScore) + " (" + changeDetail(cur.allocScore - base.allocScore, base.allocScore) + + ")"; + } + + private static String allocAbsolute(Metric metric) { + return metric == null || metric.allocScore == null ? "—" : formatBytes(metric.allocScore); + } + + private static String changeDetail(double delta, double base) { + double pct = delta / base * 100.0d; + String icon = ""; + if (pct > 0.05d) { + icon = UP + " "; + } else if (pct < -0.05d) { + icon = DOWN + " "; + } + return String.format(Locale.ROOT, "%s%+.1f%%", icon, pct); + } + + private static boolean higherIsWorse(String mode) { + // thrpt: higher score = better; everything else JMH emits (avgt, ss, sampled) is time-like + return !"thrpt".equals(mode); + } + + private static Verdict worst(Verdict a, Verdict b) { + return a.severity >= b.severity ? a : b; + } + + // ------------------------------------------------------------------ + // Report rendering + // ------------------------------------------------------------------ + + private String renderReport( + List rows, + boolean bootstrap, + JSONObject meta, + String baselineContract, + String currentContract, + boolean contractDiffers) { + StringBuilder sb = new StringBuilder(); + + if (bootstrap) { + sb.append("## :bar_chart: Performance run — baseline not yet established\n\n"); + sb.append("No committed baseline found (or it is unreadable), so no comparison was performed.\n"); + sb.append("The CI update job will open a pull request that records this run as the initial baseline.\n"); + sb.append("Every value below becomes a tracked metric once that PR is merged.\n\n"); + } else { + sb.append("## :bar_chart: Performance vs baseline\n\n"); + sb.append(baselineHeader(meta)); + sb.append("\n\n"); + if (contractDiffers) { + sb.append("> :warning: **Execution contract differs from the baseline** — baseline ran `") + .append(baselineContract) + .append("`, this run used `") + .append(currentContract) + .append("`.\n") + .append("> Time deltas across different contracts are indicative only; refresh the ") + .append("baseline to restore a strictly valid comparison.\n\n"); + } + } + + sb.append("| Benchmark | Mode | Baseline | Current | "); + sb.append(bootstrap ? "Alloc/op | Verdict |\n" : "Δ time | Alloc/op (Δ) | Verdict |\n"); + sb.append(bootstrap ? "|---|:-:|---:|---:|---:|:-:|\n" : "|---|:-:|---:|---:|---:|---:|:-:|\n"); + + for (Row row : rows) { + Metric shown = row.current != null ? row.current : row.baseline; + sb.append("| `") + .append(shown.displayName) + .append("` | ") + .append(shown.mode) + .append(" | ") + .append(row.baseline == null ? "—" : formatScore(row.baseline.score, row.baseline.scoreUnit)) + .append(" | ") + .append(row.current == null ? "—" : formatScore(row.current.score, row.current.scoreUnit)) + .append(" | "); + if (bootstrap) { + sb.append(allocAbsolute(row.current)); + } else { + sb.append(row.timeDetail).append(" | ").append(row.allocDetail); + } + sb.append(" | ").append(verdictLabel(row)).append(" |\n"); + } + + sb.append("\n").append(summaryLine(rows, bootstrap)).append("\n"); + sb.append(legend()); + return sb.toString(); + } + + private String baselineHeader(JSONObject meta) { + StringBuilder sb = new StringBuilder(); + sb.append("Baseline: "); + if (meta == null) { + sb.append("`").append(DEFAULT_BASELINE).append("` (no metadata)"); + } else { + String sha = meta.getString("gitSha"); + sb.append("`") + .append(sha == null ? "unknown" : sha.substring(0, Math.min(7, sha.length()))) + .append("`"); + String generatedAt = meta.getString("generatedAt"); + if (generatedAt != null) { + sb.append(" · recorded ").append(generatedAt); + } + String jdk = meta.getString("jdkVersion"); + if (jdk != null) { + sb.append(" · ").append(jdk); + } + String runner = meta.getString("runnerLabel"); + if (runner != null) { + sb.append(" · ").append(runner); + } + } + sb.append(String.format(Locale.ROOT, " · thresholds: warn ≥ %.0f%% / fail ≥ %.0f%%", warnPct, failPct)); + return sb.toString(); + } + + private String summaryLine(List rows, boolean bootstrap) { + int ok = 0; + int improved = 0; + int warn = 0; + int fail = 0; + int missing = 0; + int newCount = 0; + for (Row row : rows) { + switch (row.verdict) { + case OK: + ok++; + break; + case IMPROVED: + improved++; + break; + case WARN: + warn++; + break; + case FAIL: + fail++; + break; + case MISSING: + missing++; + break; + case NEW: + newCount++; + break; + default: + break; + } + } + + StringBuilder sb = new StringBuilder("**Summary**: "); + if (bootstrap) { + sb.append(newCount).append(" metric(s) recorded, none compared yet"); + return sb.toString(); + } + sb.append(ok) + .append(" ") + .append(OK) + .append(" · ") + .append(improved) + .append(" ") + .append(IMPROVED) + .append(" · ") + .append(warn) + .append(" ") + .append(WARN) + .append(" · ") + .append(fail) + .append(" ") + .append(FAIL); + if (missing > 0) { + sb.append(" · ").append(missing).append(" ").append(MISSING); + } + if (newCount > 0) { + sb.append(" · ").append(newCount).append(" ").append(NEW); + } + + boolean gateFailed = fail > 0 || (missing > 0 && !allowMissing); + sb.append(" — gate: **").append(gateFailed ? "FAILED" : "PASSED").append("**"); + if (gateFailed) { + sb.append(" (regression beyond fail threshold or missing benchmark)"); + } + return sb.toString(); + } + + private String legend() { + return "\n

\nVerdict legend & notes\n\n" + + "- " + + OK + + ": within the warn threshold; " + + IMPROVED + + ": at least the warn threshold faster.\n" + + "- " + + WARN + + ": slower beyond the warn threshold, or beyond the fail threshold but still within JMH " + + "statistical error bars (noisy CI runner) — human judgement required.\n" + + "- " + + FAIL + + ": slower beyond the fail threshold with non-overlapping error bars — fails the CI run.\n" + + "- " + + NEW + + ": not tracked by the baseline yet; becomes tracked at the next baseline refresh. " + + MISSING + + ": tracked benchmark absent from the current run.\n" + + "- Time is the JMH average per operation; Δ alloc compares `gc.alloc.rate.norm` " + + "(bytes allocated per operation) — a very stable regression signal.\n" + + "- To deliberately accept a performance change, refresh the baseline: " + + "*Actions → Benchmark → Run workflow* with **update_baseline** checked " + + "(see `fesod-benchmark/benchmark.md`).\n" + + "\n
\n"; + } + + private static String verdictLabel(Row row) { + switch (row.verdict) { + case IMPROVED: + return IMPROVED; + case WARN: + return WARN; + case FAIL: + return FAIL; + case NEW: + return NEW; + case MISSING: + return MISSING; + default: + return OK; + } + } + + // ------------------------------------------------------------------ + // JMH JSON parsing + // ------------------------------------------------------------------ + + private static List readResults(File file) throws IOException { + String content = new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8); + JSONArray array = JSON.parseArray(content); + List results = new ArrayList(); + for (int i = 0; i < array.size(); i++) { + JSONObject entry = array.getJSONObject(i); + JSONObject primary = entry.getJSONObject("primaryMetric"); + if (primary == null) { + continue; + } + JSONObject params = entry.getJSONObject("params"); + + Double allocScore = null; + double allocError = 0.0d; + String allocUnit = null; + JSONObject secondary = entry.getJSONObject("secondaryMetrics"); + if (secondary != null) { + JSONObject alloc = secondary.getJSONObject(GC_ALLOC_RATE_NORM); + if (alloc != null && alloc.containsKey("score")) { + allocScore = sanitize(alloc.getDouble("score")); + allocError = sanitize(alloc.containsKey("scoreError") ? alloc.getDouble("scoreError") : 0.0d); + allocUnit = alloc.getString("scoreUnit"); + } + } + + String benchmark = entry.getString("benchmark"); + results.add(new Metric( + keyOf(benchmark, params), + displayNameOf(benchmark, params), + entry.getString("mode"), + primary.getDoubleValue("score"), + sanitize(primary.containsKey("scoreError") ? primary.getDouble("scoreError") : 0.0d), + primary.getString("scoreUnit"), + allocScore, + allocError, + allocUnit)); + } + return results; + } + + private static JSONObject readMeta(File file) { + if (!file.isFile()) { + return null; + } + try { + return JSON.parseObject(new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8)); + } catch (IOException e) { + return null; + } + } + + /** + * Reads the JMH execution contract (forks, warmup, measurement) from the first result + * entry, e.g. {@code "3 forks, 3x1 s warmup, 5x2 s measurement"}. Comparisons are only + * strictly valid when the baseline and the current run share the same contract. + */ + private static String readContract(File file) throws IOException { + JSONArray array = JSON.parseArray(new String(Files.readAllBytes(file.toPath()), StandardCharsets.UTF_8)); + if (array.isEmpty()) { + return null; + } + JSONObject entry = array.getJSONObject(0); + return String.format( + Locale.ROOT, + "%d fork%s, %dx%s warmup, %dx%s measurement", + entry.getIntValue("forks"), + entry.getIntValue("forks") == 1 ? "" : "s", + entry.getIntValue("warmupIterations"), + entry.getString("warmupTime"), + entry.getIntValue("measurementIterations"), + entry.getString("measurementTime")); + } + + private static Map index(List metrics) { + Map byKey = new TreeMap(); + for (Metric metric : metrics) { + byKey.put(metric.key, metric); + } + return byKey; + } + + private static String keyOf(String benchmark, JSONObject params) { + return benchmark + paramsSuffix(params); + } + + private static String displayNameOf(String benchmark, JSONObject params) { + String simple = benchmark.substring(benchmark.lastIndexOf('.') + 1); + return simple + paramsSuffix(params); + } + + private static String paramsSuffix(JSONObject params) { + if (params == null || params.isEmpty()) { + return ""; + } + List parts = new ArrayList(); + for (String key : new TreeSet(params.keySet())) { + parts.add(key + "=" + params.getString(key)); + } + return " [" + String.join(", ", parts) + "]"; + } + + /** JMH emits "NaN" errors for short runs; treat those as "no error information". */ + private static double sanitize(Double value) { + if (value == null || value.isNaN() || value.isInfinite()) { + return 0.0d; + } + return value; + } + + /** Human-readable byte size for gc.alloc.rate.norm values (bytes per op). */ + private static String formatBytes(double bytes) { + if (bytes >= 1024.0d * 1024.0d * 1024.0d) { + return String.format(Locale.ROOT, "%.1f GB", bytes / (1024.0d * 1024.0d * 1024.0d)); + } + if (bytes >= 1024.0d * 1024.0d) { + return String.format(Locale.ROOT, "%.1f MB", bytes / (1024.0d * 1024.0d)); + } + if (bytes >= 1024.0d) { + return String.format(Locale.ROOT, "%.0f KB", bytes / 1024.0d); + } + return String.format(Locale.ROOT, "%.0f B", bytes); + } + + private static String formatScore(double score, String unit) { + String number; + double abs = Math.abs(score); + if (abs >= 100.0d) { + number = String.format(Locale.ROOT, "%.0f", score); + } else if (abs >= 10.0d) { + number = String.format(Locale.ROOT, "%.1f", score); + } else { + number = String.format(Locale.ROOT, "%.2f", score); + } + return unit == null || unit.isEmpty() ? number : number + " " + unit; + } + + private static void usageAndExit(String message) { + System.err.println("ERROR: " + message); + System.err.println(); + System.err.println("Usage: BaselineComparator [--baseline ] [--baseline-meta ]"); + System.err.println(" --current [--report ]"); + System.err.println(" [--warn ] [--fail ] [--allow-missing]"); + System.exit(2); + } +} diff --git a/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/baseline/BaselineRunner.java b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/baseline/BaselineRunner.java new file mode 100644 index 000000000..d3c29fdc0 --- /dev/null +++ b/fesod-benchmark/src/main/java/org/apache/fesod/sheet/benchmark/baseline/BaselineRunner.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.fesod.sheet.benchmark.baseline; + +import java.io.File; +import org.apache.fesod.sheet.benchmark.FesodBenchmark; +import org.openjdk.jmh.results.format.ResultFormatType; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import org.openjdk.jmh.runner.options.TimeValue; + +/** + * Single entry point for the performance baseline suite used by the regression CI. + * + *

Runs {@link FesodBenchmark} with a fixed execution contract (JVM settings, forks, + * iterations, gc profiler) so that a run recorded today is comparable with the committed + * baseline. Every knob can be overridden with system properties for local experiments, + * but CI always uses the defaults: + * + * 

+ *   benchmark.forks                   (default 3)
+ *   benchmark.warmup.iterations       (default 3)
+ *   benchmark.warmup.seconds          (default 1)
+ *   benchmark.measurement.iterations  (default 5)
+ *   benchmark.measurement.seconds     (default 2)
+ *   benchmark.datasetSizes            (default SMALL,MEDIUM,LARGE)
+ *   benchmark.fileFormats             (default XLSX,CSV)
+ *   benchmark.result                  (default target/baseline-current.json)
+ *
+ * + *

Usage with the shaded jar: + * 

+ *   java -cp fesod-benchmark/target/benchmarks.jar \
+ *       org.apache.fesod.sheet.benchmark.baseline.BaselineRunner
+ *
+ * + *

The gc profiler is always enabled: allocation per operation is far less noisy than + * wall-clock time on shared CI runners and is tracked in the baseline as a second signal. + */ +public final class BaselineRunner { + + private static final String[] FIXED_JVM_ARGS = {"-Xms1g", "-Xmx1g", "-XX:+UseG1GC"}; + + private BaselineRunner() {} + + public static void main(String[] args) throws RunnerException { + int forks = intProperty("benchmark.forks", 3); + int warmupIterations = intProperty("benchmark.warmup.iterations", 3); + int warmupSeconds = intProperty("benchmark.warmup.seconds", 1); + int measurementIterations = intProperty("benchmark.measurement.iterations", 5); + int measurementSeconds = intProperty("benchmark.measurement.seconds", 2); + String[] datasetSizes = + property("benchmark.datasetSizes", "SMALL,MEDIUM,LARGE").split(","); + String[] fileFormats = property("benchmark.fileFormats", "XLSX,CSV").split(","); + String resultFile = property("benchmark.result", "target/baseline-current.json"); + + printEnvironment(forks, warmupIterations, warmupSeconds, measurementIterations, measurementSeconds, resultFile); + + File resultParent = new File(resultFile).getAbsoluteFile().getParentFile(); + if (resultParent != null && !resultParent.exists()) { + resultParent.mkdirs(); + } + + Options opt = new OptionsBuilder() + .include(FesodBenchmark.class.getSimpleName()) + .param("datasetSize", trim(datasetSizes)) + .param("fileFormat", trim(fileFormats)) + .forks(forks) + .warmupIterations(warmupIterations) + .warmupTime(TimeValue.seconds(warmupSeconds)) + .measurementIterations(measurementIterations) + .measurementTime(TimeValue.seconds(measurementSeconds)) + .addProfiler("gc") + .shouldFailOnError(true) + .jvmArgs(FIXED_JVM_ARGS) + .result(resultFile) + .resultFormat(ResultFormatType.JSON) + .build(); + + new Runner(opt).run(); + + System.out.println(); + System.out.println("====================================================="); + System.out.println("Baseline run finished. Results written to: " + resultFile); + System.out.println("Compare with the committed baseline via BaselineComparator."); + System.out.println("====================================================="); + } + + private static void printEnvironment( + int forks, + int warmupIterations, + int warmupSeconds, + int measurementIterations, + int measurementSeconds, + String resultFile) { + System.out.println("====================================================="); + System.out.println("Fesod baseline suite"); + System.out.println(" JVM : " + System.getProperty("java.vm.name") + " " + + System.getProperty("java.version") + " (" + System.getProperty("java.vendor") + ")"); + System.out.println(" OS : " + System.getProperty("os.name") + " " + System.getProperty("os.version") + + " " + System.getProperty("os.arch")); + System.out.println(" Forks : " + forks); + System.out.println(" Warmup : " + warmupIterations + " x " + warmupSeconds + "s"); + System.out.println(" Measurement: " + measurementIterations + " x " + measurementSeconds + "s"); + System.out.println(" JVM args : " + String.join(" ", FIXED_JVM_ARGS)); + System.out.println(" Profiler : gc"); + System.out.println(" Result : " + resultFile); + System.out.println("====================================================="); + } + + private static String[] trim(String[] values) { + String[] trimmed = new String[values.length]; + for (int i = 0; i < values.length; i++) { + trimmed[i] = values[i].trim(); + } + return trimmed; + } + + private static String property(String name, String defaultValue) { + String value = System.getProperty(name); + return value == null || value.trim().isEmpty() ? defaultValue : value.trim(); + } + + private static int intProperty(String name, int defaultValue) { + String value = System.getProperty(name); + if (value == null || value.trim().isEmpty()) { + return defaultValue; + } + try { + return Integer.parseInt(value.trim()); + } catch (NumberFormatException e) { + System.out.println("WARN: cannot parse -D" + name + "=" + value + ", using default " + defaultValue); + return defaultValue; + } + } +}