diff --git a/docs/fuzzing.md b/docs/fuzzing.md index 70fe9d70..2154d1b7 100644 --- a/docs/fuzzing.md +++ b/docs/fuzzing.md @@ -64,6 +64,9 @@ flag `-f` to stop after the first error. | `--gen_bound` | Whether generated trees should try to reuse an existing bound name instead of a fresh one, so lookups get exercised too (default on). | | `--weights [ ...]` | Bias generation towards or away from specific tokens; see [Token weights](#token-weights). | | `-l, --log_level` | `Trace`, `Debug`, `Info`, `Warning`, `Output`, `Error`, or `None`. `Trace` prints every generated tree and its output, not just failures. | +| `--samples` | Files with sample programs to use in program generation. | +| `--sampling-level` | Number of levels sliced from a sample tree in program generation. Defaults to 1. | +| `--sampling-frequency` | Chance of preferring a sample node over random generation (0-100). Defaults to 50. | ## Reproducing a failure @@ -107,6 +110,22 @@ This samples the random number generator itself across `seed_count` seeds and reports the entropy of the values produced. This is a sanity check on randomness, not on any WF spec so most users won't need it. +## Using samples + +Using sample programs during fuzzing increases the likelihood of generating valid trees or valid subtrees. This is especially helpful when fuzzing early passes, which typically operate on less structured trees. Sampling has been shown to improve code coverage of fuzzing across several Trieste projects. + +To run the fuzzer with samples, provide one or more sample files: + +``` +my_tool test --samples examples/*.my_ext +``` + +When sampling is enabled, the fuzzer parses the sample files and may run them through all passes up to the selected starting pass. Each sample tree, and all of its subtrees, is stored in a map keyed by the type of its root node (token). + +Starting from the `Top` node, we generate the tree top-down: for each node, we choose its children by randomly picking one of the valid tokens (or token sequences) defined in the well-formedness specification. If sampling is enabled, we may instead copy a randomly chosen node of that type from the sample map, based on the sampling frequency (50% by default). + +The `--sampling-level` flag controls how much of a sampled subtree is copied. At the default level of 1, we copy the sampled node's children but not their descendants. At higher levels, we copy descendants down to that depth, or the whole subtree if it is shallower. A level of 0 copies the entire sampled subtre; to avoid reproducing complete sample programs, the `Top` node's children are then always generated from the well-formedness specification. + ## Related: pattern checking The `check` subcommand performs a separate, static check: it looks for diff --git a/include/trieste/driver.h b/include/trieste/driver.h index 50061302..507765d3 100644 --- a/include/trieste/driver.h +++ b/include/trieste/driver.h @@ -21,9 +21,8 @@ namespace trieste CLI::App app; Options* options; - public: - Driver(const Reader& reader_, Options* options_ = nullptr) - : reader(reader_), app(reader_.language_name()), options(options_) +public : Driver(const Reader& reader_, Options* options_ = nullptr) +: reader(reader_), app(reader_.language_name()), options(options_) {} Driver( @@ -193,8 +192,26 @@ namespace trieste check->add_option( "-i,--ignore_token", ignored_tokens, - "Ignore this token when checking patterns against well-formedness " - "rules."); + "Ignore this token when checking patterns against well-formedness rules."); + + std::vector sample_files; + test->add_option( + "--samples", + sample_files, + "Files to extract sample nodes from for fuzz testing"); + + size_t sampling_level = 1; + test->add_option( + "--sampling-level", + sampling_level, + "Level of sampling to use for selecting sample nodes"); + + size_t sampling_frequency = 50; // Default to 50% sampling frequency + test->add_option( + "--sampling-frequency", + sampling_frequency, + "Frequency of using sampled nodes over randomly generated nodes during " + "testing (0-100)"); try { @@ -263,6 +280,8 @@ namespace trieste } else if (*test) { + Nodes sample_trees; + if (pass_names_no_parse.empty()) { logging::Error() << "No passes available for testing." << std::endl; @@ -298,6 +317,31 @@ namespace trieste test_end_pass = test_start_pass; } + // Sort the sample files so they are loaded in a deterministic + // order making fuzzing reproducible. + std::sort(sample_files.begin(), sample_files.end()); + + for (auto const& sample_path : sample_files) + { + if (!(std::filesystem::is_regular_file(sample_path) || + std::filesystem::is_symlink(sample_path))) + { + logging::Output() << "Not sampling " << sample_path << std::endl; + continue; + } + + Node sample_program = reader.parser().parse(sample_path); + + if (!sample_program) + { + logging::Error() << "Failed to parse test program from " + << sample_path << std::endl; + return 1; + } + + sample_trees.push_back(sample_program); + } + Fuzzer fuzzer = Fuzzer(reader) .max_retries( @@ -311,7 +355,11 @@ namespace trieste .bound_vars(bound_vars) .token_weights(std::move(parsed_weights)) .test_sequence(test_sequence) - .size_stats(test_size_stats); + .size_stats(test_size_stats) + .sample_trees(sample_trees) + .sampling_level(sampling_level) + .sampling_enabled(!sample_trees.empty()) + .sampling_frequency(sampling_frequency); if (*entropy) { @@ -322,6 +370,7 @@ namespace trieste return fuzzer.test(); } } + else if (*check) { if (pass_names_no_parse.empty()) diff --git a/include/trieste/fuzzer.h b/include/trieste/fuzzer.h index cede0da2..c39affb0 100644 --- a/include/trieste/fuzzer.h +++ b/include/trieste/fuzzer.h @@ -24,6 +24,11 @@ namespace trieste size_t max_retries_; bool bound_vars_; wf::TokenWeights token_weights_; + std::vector sample_trees_; + std::map> sampled_nodes_; + size_t sampling_level_; + bool sampling_enabled_; + size_t sampling_frequency_; bool test_sequence_; bool size_stats_; @@ -217,7 +222,11 @@ namespace trieste context.current_seed, max_depth_, bound_vars_, - token_weights_); + token_weights_, + sampled_nodes_, + sampling_level_, + sampling_enabled_, + sampling_frequency_); size_t hash = ast->hash(); auto it = context.ast_hashes.find(hash); @@ -230,7 +239,11 @@ namespace trieste context.current_seed, max_depth_, bound_vars_, - token_weights_); + token_weights_, + sampled_nodes_, + sampling_level_, + sampling_enabled_, + sampling_frequency_); hash = ast->hash(); context.retries++; it = context.ast_hashes.find(hash); @@ -350,7 +363,7 @@ namespace trieste if (!logging::Trace::active()) { // We haven't printed what failed with Trace earlier, so do it - // now. Regenerate the start Ast for the error message. + // now. Regenerate the initial AST for the error message. err << "============" << std::endl << "Pass: " << pass->name() << ", seed: " << seed_context.current_seed << std::endl @@ -359,7 +372,12 @@ namespace trieste generators_, seed_context.current_seed, max_depth_, - bound_vars_) + bound_vars_, + token_weights_, + sampled_nodes_, + sampling_level_, + sampling_enabled_, + sampling_frequency_) << "------------" << std::endl << new_ast; } @@ -483,6 +501,81 @@ namespace trieste return *std::max_element(v.begin(), v.end()); } + void update_sample_nodes(wf::Wellformed wf, Pass& pass) + { + auto it = sampled_nodes_.find(Top); + if (it == sampled_nodes_.end()) + return; + + // Make a local copy of the Top sample programs so we can clear and + // repopulate `sampled_nodes_` without mutating the container we're + // iterating over. + Nodes sample_progs = it->second; + sampled_nodes_.clear(); + std::vector errors; + + for (auto& node : sample_progs) + { + if (!node) + { + continue; + } + auto [node_updated, cn, ch] = pass->run(node); + + if (!node_updated) + continue; + + // Don't use invalid updated nodes. + auto ok = wf.build_st(node_updated); + if (!ok || !wf.check(node_updated)) + { + continue; + } + + node_updated->get_errors(errors); + if (!errors.empty()) + { + errors.clear(); + continue; + } + + // Repopulate sample nodes map for next round. + node_updated->traverse([&](auto& n) { + if (n != Error) + sampled_nodes_[n->type()].emplace_back(n); + return true; + }); + } + } + + void initialize_samples(WFContext& context) + { + logging::Debug() << "Processing sample trees up to pass: " + << passes_.at(start_index_ - 1)->name() << std::endl; + for (auto& sample_tree : sample_trees_) + { + sample_tree->traverse([&](auto& n) { + sampled_nodes_[n->type()].emplace_back(n); + return true; + }); + } + + for (size_t i = 1; i < start_index_; i++) + { + auto& pass = passes_.at(i - 1); + auto& wf = pass->wf(); + auto& prev = i > 1 ? passes_.at(i - 2)->wf() : *input_wf_; + + context.push_back(prev); + context.push_back(wf); + + update_sample_nodes(wf, pass); + + context.pop_front(); + context.pop_front(); + } + } + public: Fuzzer() {} @@ -502,6 +595,10 @@ namespace trieste max_retries_(100), bound_vars_(true), token_weights_({}), + sampled_nodes_({}), + sampling_level_(0), + sampling_enabled_(false), + sampling_frequency_(50), test_sequence_(false), size_stats_(false) {} @@ -519,6 +616,29 @@ namespace trieste : Fuzzer(rewriter.passes(), rewriter.input_wf(), generators) {} + std::vector pass_names() const + { + std::vector names; + std::transform( + passes_.begin(), + passes_.end(), + std::back_inserter(names), + [](const auto& pass) { return pass->name(); }); + return names; + } + + // Returns the 1-based index of the named pass (matching the start_index_/ + // end_index_ convention), or size_t max if no such pass exists. + size_t pass_index(const std::string& name) const + { + for (size_t i = 0; i < passes_.size(); i++) + { + if (passes_[i]->name() == name) + return i + 1; + } + return std::numeric_limits::max(); + } + size_t max_depth() const { return max_depth_; @@ -623,6 +743,46 @@ namespace trieste return *this; } + bool bound_vars() const + { + return bound_vars_; + } + + Fuzzer& bound_vars(bool gen_bound_vars) + { + bound_vars_ = gen_bound_vars; + return *this; + } + + std::vector sample_trees() const + { + return sample_trees_; + } + + Fuzzer& sample_trees(Nodes& sample_trees) + { + sample_trees_ = sample_trees; + return *this; + } + + Fuzzer& sampling_level(size_t sampling_level) + { + sampling_level_ = sampling_level; + return *this; + } + + Fuzzer& sampling_enabled(bool sampling_enabled) + { + sampling_enabled_ = sampling_enabled; + return *this; + } + + Fuzzer& sampling_frequency(size_t sampling_frequency) + { + sampling_frequency_ = sampling_frequency; + return *this; + } + int debug_entropy() { const uint8_t NO_BYTES = 4; @@ -691,11 +851,6 @@ namespace trieste return 0; } - Fuzzer& bound_vars(bool gen_bound_vars) - { - bound_vars_ = gen_bound_vars; - return *this; - } Fuzzer& token_weights(wf::TokenWeights token_weights) { @@ -716,6 +871,13 @@ namespace trieste int ret = 0; + // Get sampled trees up to the starting pass and populate `sampled_nodes_` + // for generation. + if (sampling_enabled_ && !sample_trees_.empty()) + { + initialize_samples(context); + } + for (size_t i = start_index_; i <= end_index_; i++) { auto& pass = passes_.at(i - 1); @@ -805,6 +967,10 @@ namespace trieste } pass_stats.log(size_stats_); + if (i != end_index_) + { + update_sample_nodes(wf, pass); + } context.pop_front(); context.pop_front(); diff --git a/include/trieste/wf.h b/include/trieste/wf.h index 4bd18395..778773d5 100644 --- a/include/trieste/wf.h +++ b/include/trieste/wf.h @@ -28,6 +28,27 @@ namespace trieste using TokenWeights = std::map; using SymtabKeys = std::pair, size_t>; + struct Sampling + { + std::map& sampled_nodes; + size_t + sampling_level; // 0: subtree sampling, 1..n: subtree height sampling + bool sampling_enabled; + size_t sampling_period; // use a sample roughly 1-in-`sampling_period` + // draws; 0 = never + + Sampling( + std::map& sampled_nodes_, + size_t sampling_level_, + bool sampling_enabled_, + size_t sampling_frequency_) + : sampled_nodes(sampled_nodes_), + sampling_level(sampling_level_), + sampling_enabled(sampling_enabled_), + sampling_period( + sampling_frequency_ > 0 ? std::floor(100.0 / sampling_frequency_) : 0) + {} + }; struct Gen { TokenTerminalDistance token_terminal_distance; @@ -37,8 +58,9 @@ namespace trieste size_t ceiling_depth; double alpha; TokenWeights token_weights_; - std::map, size_t>> binding_keys; + std::map binding_keys; bool gen_bound_vars; + Sampling sampling; std::map*, std::vector> weighted_offsets_cache_; @@ -63,6 +85,10 @@ namespace trieste size_t target_depth_, std::map binding_keys_, bool gen_bound_vars_, + std::map& sampled_nodes, + size_t sampling_level, + bool sampling_enabled_, + size_t sampling_frequency_, double alpha_ = 1, size_t ceiling_multiplier_ = 2) : token_terminal_distance(token_terminal_distance_), @@ -72,7 +98,12 @@ namespace trieste ceiling_depth(ceiling_multiplier_ * target_depth), alpha(alpha_), binding_keys(binding_keys_), - gen_bound_vars(gen_bound_vars_) + gen_bound_vars(gen_bound_vars_), + sampling(Sampling( + sampled_nodes, + sampling_level, + sampling_enabled_, + sampling_frequency_)) { // Warm up RNG for (int i = 0; i < 10; i++) @@ -273,23 +304,83 @@ namespace trieste return rand(); } + bool has_sample_nodes(Token t) + { + return sampling.sampled_nodes.find(t) != sampling.sampled_nodes.end(); + } + + size_t sampling_level() + { + return sampling.sampling_level; + } + + bool sampling_enabled() + { + return sampling.sampling_enabled; + } + + bool use_sample() + { + // A frequency of 0 (period 0) means never sample + if (sampling.sampling_period == 0) + return false; + return next() % sampling.sampling_period == 0; + } + + bool subtree_sampling() + { + return sampling.sampling_level == 0; + } + private: - Nodes get_symbols(Token type, Node scope) + // Get all symbols in scope matching the binding type + Nodes get_symbols_from_type(Token type, Node st) { Nodes symbols; - if (scope) + while (st) { - // Look up symbols in the current scope that match the binding type - scope->get_symbols(symbols, [&](auto& n) { + st->get_symbols(symbols, [&](auto& n) { auto it = binding_keys.find(n->type()); return ( n->type() & flag::lookup && it != binding_keys.end() && contains_type(it->second.first, type)); }); + st = st->scope(); } return symbols; } + // Get all symbols in scope matching the binding type and location + Nodes get_symbols_from_loc(Location loc, Node st) + { + Nodes symbols; + while (st) + { + st->get_symbols( + loc, symbols, [&](auto& n) { return (n->type() & flag::lookup); }); + st = st->scope(); + } + return symbols; + } + + // Checks if child is a symbol table key for the parent type + bool + is_symtab_key(size_t child_index, Token child_type, Token parent_type) + { + auto it = binding_keys.find(parent_type); + if (it == binding_keys.end()) + return false; + + std::vector key_types = it->second.first; + auto key_field_index = it->second.second; + + if (key_types.empty()) + return false; + + return key_field_index == child_index && + contains_type(key_types, child_type); + } + // Returns true if we should generate a bound variable bool should_gen_bound(Node parent, Token child_type) { @@ -303,7 +394,7 @@ namespace trieste if (contains_type(binds_to.first, child_type)) bound_nodes.push_back(parent_type); } - if (bound_nodes.empty()) + if (bound_nodes.empty()) // No bound nodes to use { return false; } @@ -313,30 +404,119 @@ namespace trieste { return true; // Not at binding site } - // If the node we are generating is the binding key of the - // parent node, check that it is not the binding key itself - auto key_field_index = binding_keys[parent->type()].second; - auto child_index = parent->size() - 1; // child is already added - return key_field_index != child_index || - !(parent->type() & flag::shadowing); + // If the node we are generating is the binding key of the parent node, + // we should not use a bound name + auto child_index = parent->size() - 1; + return !( + is_symtab_key(child_index, child_type, parent->type()) && + parent->type() & flag::shadowing); + } + + bool is_in_scope(Location loc, Node scope) + { + auto symbols = get_symbols_from_loc(loc, scope); + return symbols.size() != 0; } public: - Location location(Node parent, Node child) + Location fresh_location(Node child) { - auto type = child->type(); - Nodes symbols = get_symbols(type, parent->scope()); + return gloc(rand, child); + } - // Ensure we don't always use bindings when they exist by adding +1 + // Generate location for child node, preferring to reuse existing + // locations of in-scope symbols of the same type if gen_bound_vars is + // enabled and the child is not a binding key + Location gen_location(Node parent, Node child) + { + auto type = child->type(); + Nodes symbols = get_symbols_from_type(type, child->scope()); + // Adding +1 to allow for fresh location with a small probability auto rand_symbol = static_cast(next() % (symbols.size() + 1)); - // Prefer location from a symbol table + // Prefer location from a symbol table if gen_bound_vars is enabled. if (rand_symbol < symbols.size() && should_gen_bound(parent, type)) { - auto key_index = binding_keys[type].second; - return (symbols[rand_symbol]->at(key_index))->location(); + auto sym = symbols[rand_symbol]; + auto key_index = binding_keys.at(sym->type()).second; + return (sym->at(key_index))->location(); } - // Use fresh location - return gloc(rand, child); + auto fresh = fresh_location(child); + while (is_in_scope(fresh, parent->scope())) + { + // "Fresh" location might be in scope if the fresh machinery has + // been used in both rewriting and generation + fresh = fresh_location(child); + } + return fresh; + } + + // Clone node without children, + // giving it a fresh location if it's a symtab key where shadowing is + // disallowed and the symbol is already in scope. + Node non_shadow_clone(Node parent, Node node) + { + Node node_clone = NodeDef::create(node->type()); + parent->push_back(node_clone); + auto node_index = parent->size() - 1; + Location location; + + if ( + is_symtab_key(node_index, node->type(), parent->type()) && + parent->type() & flag::shadowing && + is_in_scope(node->location(), parent->scope())) + { + location = fresh_location(node_clone); + } + else + { + // Reuse location if not a shadowing symtab key in scope + location = node->location(); + } + if (is_symtab_key(node_index, node->type(), parent->type())) + { + // All nodes that serves as symtab keys should be bound so we can + // look them up during generation + parent->bind(location); + } + node_clone->set_location(location); + return node_clone; + } + + // Clone depth layers of parent tree + // push all incomplete subtrees to continuations vector + void + clone_depth(size_t depth, Node parent, Node node, Nodes& continuations) + { + Node node_clone = non_shadow_clone(parent, node); + + if (depth == 0) + { + continuations.push_back(node_clone); + } + else + { + for (auto& child : *node) + { + clone_depth(depth - 1, node_clone, child, continuations); + } + } + } + + void gen_sample(Node& node, Nodes& continuations) + { + auto& samples = sampling.sampled_nodes[node->type()]; + size_t choice = rand() % samples.size(); + auto sample = samples[choice]; + + // sampling_level 0 means clone the entire subtree; otherwise clone that + // many levels and leave the rest to be generated. + constexpr size_t full_subtree = std::numeric_limits::max(); + size_t levels = + sampling.sampling_level == 0 ? full_subtree : sampling.sampling_level; + + // Copy children from sample + for (auto& child : *sample) + clone_depth(levels, node, child, continuations); } }; struct Choice @@ -415,7 +595,7 @@ namespace trieste // the time we call g.location(). auto child = NodeDef::create(type); node->push_back(child); - child->set_location(g.location(node, child)); + child->set_location(g.gen_location(node, child)); } }; @@ -664,7 +844,9 @@ namespace trieste { field.choice.gen(g, depth, node); if (binding == field.name) - node->bind(node->back()->location()); + { + node->bind(node->back()->location()); // Bind to symbol table + } } } @@ -885,9 +1067,8 @@ namespace trieste return ok; } - private: - void - populate_binding_keys(std::map& binding_keys) const + void populate_binding_keys( + std::map& binding_keys) const { for (const auto& [t, s] : shapes) { @@ -900,10 +1081,11 @@ namespace trieste if (shape.binding != Invalid) { auto key_index = shape.index(shape.binding); - binding_keys.emplace( - tok, - std::make_pair( - shape.fields.at(key_index).choice.types, key_index)); + auto& types = shape.fields.at(key_index).choice.types; + if (!types.empty()) + { + binding_keys.emplace(tok, std::make_pair(types, key_index)); + } } } }, @@ -917,28 +1099,48 @@ namespace trieste Seed seed, size_t target_depth, bool gen_bound, - TokenWeights token_weights = {}) const + TokenWeights token_weights, + std::map& sampled_nodes, + size_t sampling_level, + bool sampling_enabled, + size_t sampling_frequency) const { // Collect map of tokens to their binding token and the corresponding - // index - std::map binding_keys = {}; - if (gen_bound) + // index in the children vector. This is used to preferentially generate + // bound variables that are in scope. + auto binding_keys = std::map{}; + if (gen_bound || sampling_enabled) + { populate_binding_keys(binding_keys); + } auto g = Gen( - compute_minimum_distance_to_terminal(target_depth), - gloc, - seed, - target_depth, - binding_keys, - gen_bound) - .token_weights(std::move(token_weights)); + compute_minimum_distance_to_terminal(target_depth), + gloc, + seed, + target_depth, + binding_keys, + gen_bound, + sampled_nodes, + sampling_level, + sampling_enabled, + sampling_frequency) + .token_weights(std::move(token_weights)); auto top = NodeDef::create(Top); ast::detail::top_node() = top; gen_node(g, 0, top); return top; } + Node + gen(GenNodeLocationF gloc, Seed seed, size_t target_depth, bool gen_bound) + const + { + std::map empty_nodes; + return gen( + gloc, seed, target_depth, gen_bound, {} , empty_nodes, 0, false, 0); + } + std::size_t min_dist_to_terminal( TokenTerminalDistance& distance, const std::set& prefix, @@ -1021,11 +1223,30 @@ namespace trieste if (find == shapes.end()) return; + Nodes continuations; + if ( + g.sampling_enabled() && + depth < g.ceiling_depth && // Guarantees termination + g.has_sample_nodes(node->type()) && g.use_sample() && + !(depth == 0 && g.subtree_sampling())) // Avoid copying entire sample + { + g.gen_sample(node, continuations); + // Continue generating next layer + for (auto child : continuations) + { + gen_node(g, depth + g.sampling_level(), child); + } + return; + } + + // Generate based on WF std::visit( [&](auto& shape) { shape.gen(g, depth, node); }, find->second); - + // Continue generating next layer for (auto& child : *node) + { gen_node(g, depth + 1, child); + } } bool build_st(Node& node) const diff --git a/parsers/test/fuzz_util.h b/parsers/test/fuzz_util.h new file mode 100644 index 00000000..66b18949 --- /dev/null +++ b/parsers/test/fuzz_util.h @@ -0,0 +1,51 @@ +#pragma once + +#include +#include +#include +#include + +namespace trieste +{ + // Load sample trees for fuzz testing from `path`, which may be a single file + // or a directory walked recursively for files with extension `ext`. Parse + // failures are logged and skipped (both for single files and directory + // entries); an empty/non-existent path yields no samples. + inline Nodes load_sample_trees( + Reader& reader, const std::filesystem::path& path, std::string_view ext) + { + Nodes sample_trees; + if (path.empty() || !std::filesystem::exists(path)) + return sample_trees; + + // Collect matching paths first, then sort them so samples are loaded in a + // deterministic order, making fuzzing reproducible across filesystems. + std::vector files; + if (std::filesystem::is_directory(path)) + { + for (const auto& entry : + std::filesystem::recursive_directory_iterator(path)) + { + if (entry.is_regular_file() && entry.path().extension() == ext) + files.push_back(entry.path()); + } + } + else + { + files.push_back(path); + } + std::sort(files.begin(), files.end()); + + for (const auto& p : files) + { + if (Node sample_program = reader.parser().parse(p)) + sample_trees.push_back(sample_program); + else + logging::Error() << "Failed to parse " << p << std::endl; + } + + logging::Info() << "Collected " << sample_trees.size() + << " sample trees for fuzz testing" << std::endl; + return sample_trees; + } +} diff --git a/parsers/test/json_fuzzer.cc b/parsers/test/json_fuzzer.cc index 24826942..1fb5ddd5 100644 --- a/parsers/test/json_fuzzer.cc +++ b/parsers/test/json_fuzzer.cc @@ -1,3 +1,5 @@ +#include "fuzz_util.h" + #include #include #include @@ -38,6 +40,32 @@ int main(int argc, char** argv) "None") ->check(logging::set_log_level_from_string); + std::string pass; + app.add_option("-p,--start-pass", pass, "Test only this pass"); + + std::filesystem::path sample_files; + app.add_option( + "--samples", + sample_files, + "Files to extract sample nodes from for fuzz testing"); + + size_t sampling_level = 1; + app.add_option( + "--sampling-level", + sampling_level, + "Level of sampling to use for selecting sample nodes"); + + size_t sampling_frequency = 50; // Default to 50% sampling frequency + app.add_option( + "--sampling-frequency", + sampling_frequency, + "Frequency of using sampled nodes over randomly generated nodes during " + "testing (0-100)"); + + auto bound_vars = true; + app.add_option( + "--gen_bound", bound_vars, "Generate bound variable names if possible"); + try { app.parse(argc, argv); @@ -49,21 +77,40 @@ int main(int argc, char** argv) logging::Output() << "Testing x" << count << ", seed: " << seed << std::endl; - Fuzzer fuzzer; Reader reader = json::reader(); + Nodes sample_trees = load_sample_trees(reader, sample_files, ".json"); + + Fuzzer fuzzer; if (transform == "reader") - { fuzzer = Fuzzer(reader); - } else - { fuzzer = Fuzzer(json::writer("fuzzer"), reader.parser().generators()); + + const auto names = fuzzer.pass_names(); + size_t start_index = pass.empty() ? 1 : fuzzer.pass_index(pass); + if (start_index == std::numeric_limits::max()) + { + std::string joined; + for (const auto& n : names) + joined += (joined.empty() ? "" : ", ") + n; + logging::Error() << "Pass '" << pass << "' not in {" << joined << "}" + << std::endl; + return 1; } + size_t end_index = pass.empty() || sequence ? names.size() : start_index; + return fuzzer.start_seed(seed) + .start_index(start_index) + .end_index(end_index) .seed_count(count) .failfast(failfast) .max_retries(static_cast(count) * 2) .test_sequence(sequence) + .sample_trees(sample_trees) + .sampling_enabled(!sample_trees.empty()) + .sampling_frequency(sampling_frequency) + .sampling_level(sampling_level) + .bound_vars(bound_vars) .test(); } diff --git a/parsers/test/yaml_fuzzer.cc b/parsers/test/yaml_fuzzer.cc index d64bc908..f3f9f395 100644 --- a/parsers/test/yaml_fuzzer.cc +++ b/parsers/test/yaml_fuzzer.cc @@ -1,6 +1,8 @@ +#include "fuzz_util.h" #include "trieste/logging.h" #include +#include #include #include @@ -15,7 +17,8 @@ int main(int argc, char** argv) std::string transform; app.add_option("transform", transform, "Transform to test") ->check( - CLI::IsMember({"reader", "writer", "event_writer", "to_json", "all"})) + CLI::IsMember( + {"reader", "writer", "event_writer", "to_json", "reader_to_json"})) ->required(true); uint32_t seed = std::random_device()(); @@ -41,6 +44,33 @@ int main(int argc, char** argv) "None") ->check(logging::set_log_level_from_string); + std::string pass; + app.add_option("-p,--start-pass", pass, "Test only this pass"); + + // Sampling options + std::filesystem::path sample_files; + app.add_option( + "--samples", + sample_files, + "Files to extract sample nodes from for fuzz testing"); + + size_t sampling_level = 1; + app.add_option( + "--sampling-level", + sampling_level, + "Level of sampling to use for selecting sample nodes"); + + size_t sampling_frequency = 50; // Default to 50% sampling frequency + app.add_option( + "--sampling-frequency", + sampling_frequency, + "Frequency of using sampled nodes over randomly generated nodes during " + "testing (0-100)"); + + auto bound_vars = true; + app.add_option( + "--gen_bound", bound_vars, "Generate bound variable names if possible"); + try { app.parse(argc, argv); @@ -52,36 +82,52 @@ int main(int argc, char** argv) logging::Output() << "Testing x" << count << ", seed: " << seed << std::endl; - Fuzzer fuzzer; Reader reader = yaml::reader(); + Reader reader_tojson = yaml::reader() >>= yaml::to_json(); + + // Determine which reader to use for parsing sample files. + Reader& sample_reader = + (transform == "reader_to_json") ? reader_tojson : reader; + + Nodes sample_trees = load_sample_trees(sample_reader, sample_files, ".yaml"); + + Fuzzer fuzzer; if (transform == "reader") - { fuzzer = Fuzzer(reader); - } else if (transform == "writer") - { fuzzer = Fuzzer(yaml::writer("fuzzer"), reader.parser().generators()); - } else if (transform == "event_writer") - { fuzzer = Fuzzer(yaml::event_writer("fuzzer"), reader.parser().generators()); - } else if (transform == "to_json") - { fuzzer = Fuzzer(yaml::to_json(), reader.parser().generators()); - } - else if (transform == "all") + else if (transform == "reader_to_json") + fuzzer = Fuzzer(reader_tojson); + + const auto names = fuzzer.pass_names(); + size_t start_index = pass.empty() ? 1 : fuzzer.pass_index(pass); + if (start_index == std::numeric_limits::max()) { - std::vector passes = reader.passes(); - std::vector to_json_passes = yaml::to_json().passes(); - passes.insert(passes.end(), to_json_passes.begin(), to_json_passes.end()); - fuzzer = Fuzzer(passes, reader.parser().wf(), reader.parser().generators()); + std::string joined; + for (const auto& n : names) + joined += (joined.empty() ? "" : ", ") + n; + logging::Error() << "Pass '" << pass << "' not in {" << joined << "}" + << std::endl; + return 1; } + size_t end_index = pass.empty() || sequence ? names.size() : start_index; + return fuzzer.start_seed(seed) + .start_index(start_index) + .end_index(end_index) .seed_count(count) .failfast(failfast) - .max_retries(static_cast(count) * 2) + .max_retries(count * 2) .test_sequence(sequence) + .bound_vars(bound_vars) + .sampling_level(sampling_level) + .sampling_enabled(!sample_trees.empty()) + .sampling_frequency(sampling_frequency) + .sample_trees(sample_trees) .test(); } diff --git a/samples/infix/CMakeLists.txt b/samples/infix/CMakeLists.txt index 316a06e6..d82396d8 100644 --- a/samples/infix/CMakeLists.txt +++ b/samples/infix/CMakeLists.txt @@ -12,6 +12,7 @@ target_link_libraries(infix_trieste add_test(NAME infix COMMAND infix_trieste test -f) add_test(NAME infix_sequence COMMAND infix_trieste test -f --sequence) +add_test(NAME infix_samples COMMAND infix_trieste test -f --samples ../samples/infix/testsuite/examples/simple_infix.infix ../samples/infix/testsuite/examples/multi_ident.infix) add_test(NAME invalid_input COMMAND infix ./infix) set_property(TEST invalid_input PROPERTY WILL_FAIL On) add_test(NAME infix_check COMMAND infix_trieste check -w)