diff --git a/examples/Cargo.lock b/examples/Cargo.lock index 561c243..da5b559 100644 --- a/examples/Cargo.lock +++ b/examples/Cargo.lock @@ -496,6 +496,7 @@ dependencies = [ "fiftyone-pipeline-web", "fiftyone-pipeline-web-axum", "serde", + "serde_json", "serde_norway", "tokio", "tower", @@ -590,6 +591,8 @@ dependencies = [ "chrono", "fiftyone-pipeline-core", "fiftyone-pipeline-engines", + "serde", + "serde_json", "tempfile", ] @@ -1320,6 +1323,7 @@ dependencies = [ "fiftyone-pipeline-engines-fiftyone", "fiftyone-pipeline-web", "fiftyone-pipeline-web-axum", + "serde_json", "serde_norway", "tokio", "tower", diff --git a/examples/Cargo.toml b/examples/Cargo.toml index f5b4a9e..7851a9e 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -37,6 +37,7 @@ tower = "0.5" chrono = { version = "0.4", default-features = false, features = ["clock", "std"] } base64 = "0.22" serde = { version = "1", features = ["derive"] } +serde_json = "1" serde_norway = "0.9" csv = "1.3" diff --git a/examples/device-detection-examples/Cargo.toml b/examples/device-detection-examples/Cargo.toml index 31af0e5..acc6bbc 100644 --- a/examples/device-detection-examples/Cargo.toml +++ b/examples/device-detection-examples/Cargo.toml @@ -58,3 +58,8 @@ serde.workspace = true # CSV reader for the "20000 User Agents.csv" evidence file the offline and # performance examples can stream as an alternative to the YAML records. csv.workspace = true + +[dev-dependencies] +# The performance example's test reads back the results JSON it wrote and +# asserts on the shared schema, rather than on the printed report. +serde_json.workspace = true diff --git a/examples/device-detection-examples/src/bin/dd-onprem-performance.rs b/examples/device-detection-examples/src/bin/dd-onprem-performance.rs index dfcfc5f..4109b4d 100644 --- a/examples/device-detection-examples/src/bin/dd-onprem-performance.rs +++ b/examples/device-detection-examples/src/bin/dd-onprem-performance.rs @@ -58,6 +58,10 @@ pub struct ExampleOptions { /// The maximum number of User-Agents to load from the file. The bundled file /// holds 20,000; the test caps this so it stays fast. pub max_user_agents: usize, + /// Where to write the results JSON the nightly performance graphs read, if + /// anywhere. CI passes `--json-output `; an interactive run leaves it + /// unset and only reads the printed figures. + pub json_output: Option, } /// The outcome of one benchmark run. @@ -70,6 +74,27 @@ struct BenchmarkResult { elapsed: Duration, } +impl BenchmarkResult { + /// Detections per second across the timed phase, the headline throughput. + fn detections_per_second(&self) -> f64 { + let seconds = self.elapsed.as_secs_f64(); + if seconds > 0.0 { + self.detections as f64 / seconds + } else { + 0.0 + } + } + + /// Mean real time per detection, in milliseconds. + fn millisecs_per_detection(&self) -> f64 { + if self.detections > 0 { + self.elapsed.as_secs_f64() * 1000.0 / self.detections as f64 + } else { + 0.0 + } + } +} + // [example] /// Run the on-premise performance benchmark. /// @@ -123,6 +148,22 @@ pub fn run(options: ExampleOptions) -> Result<()> { report(&result, options.thread_count); + // The nightly performance graphs read this file. The example writes it + // itself so the figure does not depend on the wording or number formatting + // of the printed report above, which is free to change. + if let Some(json_output) = &options.json_output { + let results = examples_shared::PerformanceResults::new() + .higher_is_better("DetectionsPerSecond", result.detections_per_second()) + .lower_is_better("AvgMillisecsPerDetection", result.millisecs_per_detection()); + results.write_to(json_output).with_context(|| { + format!( + "failed to write the performance results to '{}'", + json_output.display() + ) + })?; + println!("Wrote performance results to '{}'.", json_output.display()); + } + // This benchmark uses PerformanceProfile::InMemory. The other profiles // (HighPerformance, Balanced, Default, LowMemory) trade memory, load time and // detection speed differently. The performance options documentation explains @@ -215,16 +256,8 @@ fn benchmark( /// second and the average time per detection. fn report(result: &BenchmarkResult, thread_count: usize) { let seconds = result.elapsed.as_secs_f64(); - let per_second = if seconds > 0.0 { - result.detections as f64 / seconds - } else { - 0.0 - }; - let ms_per_detection = if result.detections > 0 { - result.elapsed.as_secs_f64() * 1000.0 / result.detections as f64 - } else { - 0.0 - }; + let per_second = result.detections_per_second(); + let ms_per_detection = result.millisecs_per_detection(); println!("--- Benchmark results ---"); println!("\tThreads : {}", thread_count.max(1)); @@ -263,12 +296,14 @@ fn default_user_agents_file() -> Option { /// User-Agent files use the usual fallbacks. With either missing the example /// prints a clear message and exits successfully. fn main() -> Result<()> { - let mut args = std::env::args().skip(1); - let data_file = args + let arguments: Vec = std::env::args().skip(1).collect(); + let json_output = examples_shared::json_output_path(&arguments); + let mut positional = positional_arguments(&arguments).into_iter(); + let data_file = positional .next() .map(PathBuf::from) .or_else(examples_shared::dd_data_path); - let user_agents_file = args + let user_agents_file = positional .next() .map(PathBuf::from) .or_else(default_user_agents_file); @@ -293,9 +328,27 @@ fn main() -> Result<()> { // Two passes over 20,000 User-Agents is a steady, quick benchmark. passes: 2, max_user_agents: 20_000, + json_output, }) } +/// The arguments that are not the `--json-output` flag or its value, so the +/// data-file and User-Agent-file positions keep working whether or not CI asks +/// for a results file. +fn positional_arguments(arguments: &[String]) -> Vec { + let mut positional = Vec::new(); + let mut remaining = arguments.iter(); + while let Some(argument) = remaining.next() { + if argument == examples_shared::JSON_OUTPUT_FLAG { + // Consume the path that follows the flag. + let _ = remaining.next(); + } else if !argument.starts_with("--") { + positional.push(argument.clone()); + } + } + positional +} + #[cfg(test)] mod tests { use super::*; @@ -321,9 +374,63 @@ mod tests { thread_count: 2, passes: 1, max_user_agents: 200, + json_output: None, }) .expect("the on-premise performance example should complete"); } + + /// The results file CI publishes must be written in the shared schema, with + /// the throughput metric the performance graph is keyed on. + #[test] + fn writes_the_results_json_when_asked() { + let Some(data_file) = examples_shared::dd_data_path() else { + eprintln!("skipping: no on-premise data file found (set 51DEGREES_DD_PATH)"); + return; + }; + let Some(user_agents_file) = default_user_agents_file() else { + eprintln!("skipping: bundled User-Agent file not found"); + return; + }; + let json_output = std::env::temp_dir() + .join("dd-onprem-performance-results") + .join("results.json"); + let _ = std::fs::remove_file(&json_output); + + run(ExampleOptions { + data_file, + user_agents_file, + thread_count: 2, + passes: 1, + max_user_agents: 200, + json_output: Some(json_output.clone()), + }) + .expect("the on-premise performance example should complete"); + + let written = + std::fs::read_to_string(&json_output).expect("the results file should be written"); + let parsed: serde_json::Value = + serde_json::from_str(&written).expect("the results file should be valid JSON"); + assert!( + parsed["HigherIsBetter"]["DetectionsPerSecond"].is_number(), + "the results file should carry the graph metric, got: {written}" + ); + } + + /// The flag and its value must not be mistaken for the data-file and + /// User-Agent-file positions. + #[test] + fn the_results_flag_is_not_treated_as_a_positional() { + let arguments = vec![ + "data.hash".to_owned(), + "--json-output".to_owned(), + "results.json".to_owned(), + "agents.csv".to_owned(), + ]; + assert_eq!( + positional_arguments(&arguments), + vec!["data.hash", "agents.csv"] + ); + } } /* diff --git a/examples/examples-shared/Cargo.toml b/examples/examples-shared/Cargo.toml index 19f32ae..174197a 100644 --- a/examples/examples-shared/Cargo.toml +++ b/examples/examples-shared/Cargo.toml @@ -20,6 +20,11 @@ chrono.workspace = true # Base64 decoding for the obvious-placeholder resource-key check, which # base64-decodes the key before applying the length tests. base64.workspace = true +# The derive macro the performance-results model is serialised through. +serde.workspace = true +# JSON serialisation for the performance-results model the performance examples +# write for the nightly performance graphs. +serde_json.workspace = true [dev-dependencies] # Temporary directory trees for the find-file unit tests. diff --git a/examples/examples-shared/src/lib.rs b/examples/examples-shared/src/lib.rs index 3eaa0c6..e4410d6 100644 --- a/examples/examples-shared/src/lib.rs +++ b/examples/examples-shared/src/lib.rs @@ -49,6 +49,9 @@ //! data file and return age and Lite-tier warnings for the example to print. //! - [`get_property_as_string`]. Render any property from an element data bag to //! a display string, handling the missing and no-value cases. +//! - [`PerformanceResults`] and [`json_output_path`]. The results model the +//! performance examples emit for the nightly performance graphs, and the +//! `--json-output` argument they take the path from. //! - A set of sample evidence values (see the [`evidence`] module) covering the //! common detection paths. @@ -61,6 +64,7 @@ mod data_paths; mod endpoint; mod find_file; mod keys; +mod performance_results; mod properties; pub use data_file_check::{check_data_file, data_file_info, DATA_FILE_AGE_WARNING_DAYS}; @@ -73,6 +77,7 @@ pub use endpoint::{cloud_endpoint_from_env, CLOUD_ENDPOINT_ENV_VAR}; pub use find_file::{ find_file, find_file_from, MAX_DESCENT_DEPTH, MAX_DIRECTORIES_SCANNED, MAX_PARENT_LEVELS, }; +pub use performance_results::{json_output_path, PerformanceResults, JSON_OUTPUT_FLAG}; pub use keys::{ is_invalid_key, resource_key_from_env, CI_RESOURCE_KEY_FREE_ENV_VAR, CI_RESOURCE_KEY_PAID_ENV_VAR, RESOURCE_KEY_ENV_VAR, RESOURCE_KEY_ENV_VARS, diff --git a/examples/examples-shared/src/performance_results.rs b/examples/examples-shared/src/performance_results.rs new file mode 100644 index 0000000..7a178a8 --- /dev/null +++ b/examples/examples-shared/src/performance_results.rs @@ -0,0 +1,206 @@ +/* ********************************************************************* + * This Original Work is copyright of 51 Degrees Mobile Experts Limited. + * Copyright 2026 51 Degrees Mobile Experts Limited, Davidson House, + * Forbury Square, Reading, Berkshire, United Kingdom RG1 3EU. + * + * This Original Work is licensed under the European Union Public Licence + * (EUPL) v.1.2 and is subject to its terms as set out below. + * + * If a copy of the EUPL was not distributed with this file, You can obtain + * one at https://opensource.org/licenses/EUPL-1.2. + * + * The 'Compatible Licences' set out in the Appendix to the EUPL (as may be + * amended by the European Commission) shall be deemed incompatible for + * the purposes of the Work and the provisions of the compatibility + * clause in Article 5 of the EUPL shall not apply. + * + * If using the Work as, or as part of, a network application, by + * including the attribution notice(s) required under Article 5 of the EUPL + * in the end user terms of the application under an appropriate heading, + * such notice(s) shall fulfill the requirements of that article. + * ********************************************************************* */ + +//! The performance-results model the performance examples emit. +//! +//! The nightly performance graphs read a JSON file per configuration, in a +//! schema shared across every 51Degrees language repository: +//! +//! ```json +//! { +//! "HigherIsBetter": { "DetectionsPerSecond": 1234567.0 }, +//! "LowerIsBetter": { "AvgMillisecsPerDetection": 0.00081 } +//! } +//! ``` +//! +//! The example writes this file itself, and CI only copies it into place. CI +//! deliberately does not recover the figure by parsing an example's printed +//! output: a scraped figure is tied to the exact wording and number formatting +//! of that output, so renaming a label or changing a number format would +//! silently stop the graph updating. +//! +//! Both the Device Detection and the IP Intelligence performance examples build +//! their results through this one type, so they emit an identical structure and +//! there is a single definition of the schema to maintain. + +use std::collections::BTreeMap; +use std::io; +use std::path::{Path, PathBuf}; + +use serde::Serialize; + +/// The command-line flag the performance examples take the results path from. +pub const JSON_OUTPUT_FLAG: &str = "--json-output"; + +/// A set of performance metrics in the shared results schema. +/// +/// Metric names are the series keys on the graph, so they must stay stable +/// across runs of the same configuration. Metrics are held in a [`BTreeMap`] so +/// the serialised order is deterministic whatever order they were added in. +/// +/// # Examples +/// +/// ``` +/// use examples_shared::PerformanceResults; +/// +/// let results = PerformanceResults::new() +/// .higher_is_better("DetectionsPerSecond", 1_234_567.0) +/// .lower_is_better("AvgMillisecsPerDetection", 0.00081); +/// assert!(results.to_json().contains("\"DetectionsPerSecond\"")); +/// ``` +#[derive(Debug, Default, Clone, Serialize)] +pub struct PerformanceResults { + /// Metrics where a higher value is a better result, such as a throughput. + #[serde(rename = "HigherIsBetter", skip_serializing_if = "BTreeMap::is_empty")] + higher_is_better: BTreeMap, + /// Metrics where a lower value is a better result, such as a per-item cost. + #[serde(rename = "LowerIsBetter", skip_serializing_if = "BTreeMap::is_empty")] + lower_is_better: BTreeMap, +} + +impl PerformanceResults { + /// An empty result set. + pub fn new() -> Self { + Self::default() + } + + /// Add a metric where a higher value is a better result. + #[must_use] + pub fn higher_is_better(mut self, metric: &str, value: f64) -> Self { + self.higher_is_better.insert(metric.to_owned(), value); + self + } + + /// Add a metric where a lower value is a better result. + #[must_use] + pub fn lower_is_better(mut self, metric: &str, value: f64) -> Self { + self.lower_is_better.insert(metric.to_owned(), value); + self + } + + /// Whether any metric has been added. A results file with no metrics carries + /// no figure, so callers can treat this as an error rather than write one. + pub fn is_empty(&self) -> bool { + self.higher_is_better.is_empty() && self.lower_is_better.is_empty() + } + + /// Render the results as pretty-printed JSON in the shared schema. + /// + /// Serialisation of a map of `f64` cannot fail, so this returns the string + /// directly rather than a `Result`. + pub fn to_json(&self) -> String { + serde_json::to_string_pretty(self).expect("performance results are always serialisable") + } + + /// Write the results as JSON to `path`, creating the parent directory if it + /// does not already exist. + pub fn write_to(&self, path: &Path) -> io::Result<()> { + if let Some(parent) = path.parent() { + if !parent.as_os_str().is_empty() { + std::fs::create_dir_all(parent)?; + } + } + std::fs::write(path, format!("{}\n", self.to_json())) + } +} + +/// Take the results path from a `--json-output ` argument, if present. +/// +/// The performance examples share this so the flag behaves identically in each +/// one, which is what lets a single CI adapter run any of them. +pub fn json_output_path(arguments: Arguments) -> Option +where + Arguments: IntoIterator, + Argument: AsRef, +{ + let mut arguments = arguments.into_iter(); + while let Some(argument) = arguments.next() { + let argument = argument.as_ref(); + if argument == JSON_OUTPUT_FLAG { + return arguments.next().map(|path| PathBuf::from(path.as_ref())); + } + // Also accept the `--json-output=` spelling, which is what a shell + // user is most likely to type. + if let Some(path) = argument.strip_prefix(&format!("{JSON_OUTPUT_FLAG}=")) { + return Some(PathBuf::from(path)); + } + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn serialises_both_sections_in_the_shared_schema() { + let json = PerformanceResults::new() + .higher_is_better("DetectionsPerSecond", 1000.0) + .lower_is_better("AvgMillisecsPerDetection", 0.5) + .to_json(); + let parsed: serde_json::Value = serde_json::from_str(&json).expect("valid JSON"); + assert_eq!(parsed["HigherIsBetter"]["DetectionsPerSecond"], 1000.0); + assert_eq!(parsed["LowerIsBetter"]["AvgMillisecsPerDetection"], 0.5); + } + + #[test] + fn omits_an_empty_section() { + let json = PerformanceResults::new() + .higher_is_better("LookupsPerSecond", 42.0) + .to_json(); + assert!(json.contains("HigherIsBetter")); + assert!(!json.contains("LowerIsBetter")); + } + + #[test] + fn an_empty_result_set_is_reported_as_empty() { + assert!(PerformanceResults::new().is_empty()); + assert!(!PerformanceResults::new().higher_is_better("Any", 1.0).is_empty()); + } + + #[test] + fn reads_the_json_output_flag_in_both_spellings() { + assert_eq!( + json_output_path(["--json-output", "results.json"]), + Some(PathBuf::from("results.json")) + ); + assert_eq!( + json_output_path(["--json-output=results.json"]), + Some(PathBuf::from("results.json")) + ); + assert_eq!(json_output_path(["data.hash"]), None); + // A trailing flag with no value is not a path. + assert_eq!(json_output_path(["--json-output"]), None); + } + + #[test] + fn writes_the_file_and_creates_its_parent_directory() { + let directory = tempfile::tempdir().expect("a temporary directory"); + let path = directory.path().join("nested").join("results.json"); + PerformanceResults::new() + .higher_is_better("DetectionsPerSecond", 7.0) + .write_to(&path) + .expect("the results file should be written"); + let written = std::fs::read_to_string(&path).expect("the results file should be readable"); + assert!(written.contains("\"DetectionsPerSecond\": 7.0")); + } +} diff --git a/examples/ip-intelligence-examples/Cargo.toml b/examples/ip-intelligence-examples/Cargo.toml index 6350d60..9f610f1 100644 --- a/examples/ip-intelligence-examples/Cargo.toml +++ b/examples/ip-intelligence-examples/Cargo.toml @@ -59,3 +59,8 @@ anyhow = "1" # records from evidence.yml in the ip-intelligence-data submodule. serde_norway # is the maintained fork of the (now unmaintained) serde_yaml. serde_norway.workspace = true + +[dev-dependencies] +# The performance example's test reads back the results JSON it wrote and +# asserts on the shared schema, rather than on the printed report. +serde_json.workspace = true diff --git a/examples/ip-intelligence-examples/src/bin/ipi-onprem-performance.rs b/examples/ip-intelligence-examples/src/bin/ipi-onprem-performance.rs index 28cb65e..5ad5336 100644 --- a/examples/ip-intelligence-examples/src/bin/ipi-onprem-performance.rs +++ b/examples/ip-intelligence-examples/src/bin/ipi-onprem-performance.rs @@ -56,13 +56,21 @@ pub struct ExampleOptions { pub max_ips: usize, /// How many times each thread loops over the loaded IP set. pub iterations: usize, + /// Where to write the results JSON the nightly performance graphs read, if + /// anywhere. CI passes `--json-output `; an interactive run leaves it + /// unset and only reads the printed figures. + pub json_output: Option, } impl ExampleOptions { /// Default options, using the best-available loadable data-file tier and a /// thread count taken from the available parallelism. pub fn from_env() -> Option { - Self::for_tier(examples_shared::IpiTier::BestAvailable, 500, 4) + let options = Self::for_tier(examples_shared::IpiTier::BestAvailable, 500, 4)?; + Some(ExampleOptions { + json_output: examples_shared::json_output_path(std::env::args().skip(1)), + ..options + }) } /// Options pinned to a specific data-file tier, IP cap and iteration count. @@ -81,6 +89,7 @@ impl ExampleOptions { .unwrap_or(4), max_ips, iterations, + json_output: None, }) } } @@ -169,6 +178,27 @@ pub fn run(options: &ExampleOptions, out: &mut dyn Write) -> anyhow::Result<()> let multi = benchmark(&pipeline, &ips, options.thread_count, options.iterations); report(out, "Multi-threaded", &multi, options.thread_count)?; + // The nightly performance graphs read this file. The example writes it + // itself so the figure does not depend on the wording or number formatting + // of the printed report above, which is free to change. The multi-threaded + // pass is the headline figure for the product, so it is the one published. + if let Some(json_output) = &options.json_output { + let results = examples_shared::PerformanceResults::new() + .higher_is_better("LookupsPerSecond", multi.lookups_per_second()) + .lower_is_better("AvgMillisecsPerLookup", multi.ms_per_lookup()); + results.write_to(json_output).with_context(|| { + format!( + "failed to write the performance results to '{}'", + json_output.display() + ) + })?; + writeln!( + out, + "Wrote performance results to '{}'.", + json_output.display() + )?; + } + Ok(()) } // [example] @@ -310,6 +340,35 @@ mod tests { assert!(printed.contains("Multi-threaded result")); assert!(printed.contains("lookups/sec")); } + + /// The results file CI publishes must be written in the shared schema, with + /// the throughput metric the performance graph is keyed on. + #[test] + fn writes_the_results_json_when_asked() { + let Some(options) = ExampleOptions::for_tier(examples_shared::IpiTier::Asn, 50, 1) else { + eprintln!("no usable IP Intelligence data file or evidence file; skipping perf run"); + return; + }; + let json_output = std::env::temp_dir() + .join("ipi-onprem-performance-results") + .join("results.json"); + let _ = std::fs::remove_file(&json_output); + + let options = ExampleOptions { + json_output: Some(json_output.clone()), + ..options + }; + run(&options, &mut Vec::new()).expect("the performance example should run"); + + let written = + std::fs::read_to_string(&json_output).expect("the results file should be written"); + let parsed: serde_json::Value = + serde_json::from_str(&written).expect("the results file should be valid JSON"); + assert!( + parsed["HigherIsBetter"]["LookupsPerSecond"].is_number(), + "the results file should carry the graph metric, got: {written}" + ); + } } /*