From 88064307e506fb3ebdf2d1f298c8132958f2edb7 Mon Sep 17 00:00:00 2001 From: JFoederer <32476108+JFoederer@users.noreply.github.com> Date: Fri, 22 May 2026 14:51:17 +0200 Subject: [PATCH 01/19] add scenario target end condition (internal only) --- robotmbt/suiteprocessors.py | 52 +++++++++++++++++++++++-------------- robotmbt/suitereplacer.py | 4 +++ robotmbt/tracestate.py | 4 ++- 3 files changed, 40 insertions(+), 20 deletions(-) diff --git a/robotmbt/suiteprocessors.py b/robotmbt/suiteprocessors.py index 8e06c85..2304fe9 100644 --- a/robotmbt/suiteprocessors.py +++ b/robotmbt/suiteprocessors.py @@ -48,6 +48,9 @@ class SuiteProcessors: + BATCH_SIZE = 100 + SCENARIO_TARGET = 0 + @staticmethod def echo(in_suite: Suite) -> Suite: return in_suite @@ -97,27 +100,40 @@ def process_test_suite(self, in_suite: Suite, *, seed: str | int | bytes | bytea self._visualiser = self._init_visualiser(in_suite.name) if graph or export_graph_data else None try: # a short trace without the need for repeating scenarios is preferred - tracestate = self._search_direct_trace() - if not tracestate.coverage_reached(): - logger.debug("Direct trace not discovered. Now exploring with loops, allowing repetition of scenarios.") - tracestate = self._try_to_reach_full_coverage(allow_duplicate_scenarios=True, randomise=True, - unreached_scenarios=tracestate.unreached) - else: + direct_tracestate = self._search_direct_trace() + if direct_tracestate.coverage_reached(): # The visualiser assumes that the last trace is the final selected trace, which is not always # the case. Re-adding the selected trace to prevent the wrong path from being highlighted. - self._update_visualisation(TraceState(tracestate.prio_order)) - self._update_visualisation(tracestate) + self.tracestate = direct_tracestate + self._update_visualisation(self.tracestate) + self._update_visualisation(self.tracestate) + else: + self.tracestate = TraceState([s.src_id for s in self.scenarios]) + self.tracestate.unreached = direct_tracestate.unreached + logger.debug("Direct trace not discovered. Now exploring with loops, allowing repetition of scenarios.") + self._generate_next_batch(self.BATCH_SIZE) finally: # Draw the graph even when a timeout or user interrupt occurs if graph: self._write_visualisation(graph) if export_graph_data: self._export_graph_data(export_graph_data) - if not tracestate.coverage_reached(): + if len(self.tracestate) == 0: raise Exception("Unable to compose a consistent suite") - self._report_tracestate_wrapup(tracestate) - self.out_suite.scenarios = tracestate.get_trace() + self._report_tracestate_wrapup(self.tracestate) + self.index = 0 return self.out_suite + def next_scenario_request(self): + try: + if len(self.tracestate) <= self.index: + self._generate_next_batch(self.BATCH_SIZE) + if len(self.tracestate) > self.index: + self.out_suite.scenarios.append(self.tracestate[self.index].scenario) + self.index += 1 + self.tracestate.no_rewind += 1 + except AttributeError: + pass + def draw_graph_from_export_file(self, file_path: str, graph_style: str): self._visualiser = self._init_visualiser() if self._visualiser: @@ -263,14 +279,13 @@ def _create_suggestion_by_experience(self, tracestate_list, target_index=-1) -> not_in_target = [id for id in tracestate_list[target_index].not_in_trace if id not in never_reached] return never_reached + not_in_target + tracestate_list[target_index].covered_ids - def _try_to_reach_full_coverage(self, allow_duplicate_scenarios: bool, randomise: bool = False, - unreached_scenarios: list[int] = None) -> TraceState: - tracestate = TraceState([s.src_id for s in self.scenarios]) - if unreached_scenarios: - tracestate.unreached = unreached_scenarios + def _generate_next_batch(self, batchsize): + tracestate = self.tracestate + old_len = len(tracestate) self._update_visualisation(tracestate) - while not tracestate.coverage_reached(): - candidate_id = tracestate.next_candidate(retry=allow_duplicate_scenarios, randomise=randomise) + while (len(tracestate) < old_len + batchsize) and \ + (not tracestate.coverage_reached() or (self.SCENARIO_TARGET and len(tracestate) < self.SCENARIO_TARGET)): + candidate_id = tracestate.next_candidate(retry=True, randomise=True) if candidate_id is None: # No more candidates remaining for this level if not tracestate.can_rewind(): break @@ -303,7 +318,6 @@ def _try_to_reach_full_coverage(self, allow_duplicate_scenarios: bool, randomise logger.debug(f"last state:\n{tracestate.model.get_status_text()}") self._update_visualisation(tracestate) self._update_visualisation(tracestate) - return tracestate @staticmethod def __last_candidate_changed_nothing(tracestate: TraceState) -> bool: diff --git a/robotmbt/suitereplacer.py b/robotmbt/suitereplacer.py index bc55e95..ab7b878 100644 --- a/robotmbt/suitereplacer.py +++ b/robotmbt/suitereplacer.py @@ -185,6 +185,8 @@ def add_next_new(self, target_suite: robot.model.TestSuite): new_target = self.add_suite(new_suite, target_suite) self.add_next_new(new_target) except StopIteration: + if hasattr(self.processor_lib, 'next_scenario_request'): + self.processor_lib.next_scenario_request() try: self.add_test(next(self.test_case_gen[-1]), target_suite) except StopIteration: @@ -238,6 +240,8 @@ def _end_suite(self, suite: robot.model.TestSuite, result): def _end_test(self, test_case: robot.model.TestCase, result): if not self.mbt_anchor_suite: return + if hasattr(self.processor_lib, 'next_scenario_request'): + self.processor_lib.next_scenario_request() try: self.add_test(next(self.test_case_gen[-1]), self.current_suite) except StopIteration: diff --git a/robotmbt/tracestate.py b/robotmbt/tracestate.py index 77dc639..665d821 100644 --- a/robotmbt/tracestate.py +++ b/robotmbt/tracestate.py @@ -59,6 +59,7 @@ def __init__(self, scenario_indexes: list[int]): self._tried: list[list[int]] = [[]] # Keeps track of the scenarios already tried at each step in the trace self._snapshots: list[TraceSnapShot] = [] # Keeps details for elements in trace self._open_refinements: list[int] = [] + self.no_rewind = 0 @property def model(self) -> ModelSpace | None: @@ -122,6 +123,7 @@ def copy(self): cp._tried = [triedlist[:] for triedlist in self._tried] cp._snapshots = self._snapshots[:] cp._open_refinements = self._open_refinements[:] + cp.no_rewind = self.no_rewind return cp def coverage_reached(self) -> bool: @@ -209,7 +211,7 @@ def push_partial_scenario(self, index: int, scenario: Scenario, model: ModelSpac self._snapshots.append(TraceSnapShot(id, scenario, model, remainder, self.coverage_drought)) def can_rewind(self) -> bool: - return len(self._snapshots) > 0 + return len(self._snapshots[self.no_rewind:]) > 0 def rewind(self) -> TraceSnapShot | None: id = self._snapshots[-1].id From 1ef6ee33a52d251997472f8280330bac530553f4 Mon Sep 17 00:00:00 2001 From: JFoederer <32476108+JFoederer@users.noreply.github.com> Date: Fri, 22 May 2026 17:25:36 +0200 Subject: [PATCH 02/19] make first long run options configurable --- robotmbt/suiteprocessors.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/robotmbt/suiteprocessors.py b/robotmbt/suiteprocessors.py index 2304fe9..f9b2421 100644 --- a/robotmbt/suiteprocessors.py +++ b/robotmbt/suiteprocessors.py @@ -48,9 +48,6 @@ class SuiteProcessors: - BATCH_SIZE = 100 - SCENARIO_TARGET = 0 - @staticmethod def echo(in_suite: Suite) -> Suite: return in_suite @@ -84,6 +81,9 @@ def flatten(self, in_suite: Suite) -> Suite: return out_suite def process_test_suite(self, in_suite: Suite, *, seed: str | int | bytes | bytearray = 'new', + batch_size: str | int = 100, + coverage_target: str | int | None = 1, + scenario_target: str | int | None = None, graph: str = '', export_graph_data: str = '') -> Suite: self.out_suite = Suite(in_suite.name) self.out_suite.filename = in_suite.filename @@ -96,8 +96,16 @@ def process_test_suite(self, in_suite: Suite, *, seed: str | int | bytes | bytea logger.debug("Use these numbers to reference scenarios from traces\n\t" + "\n\t".join([f"{s.src_id}: {s.name}" for s in self.scenarios])) + # handle options + self.batch_size = int(batch_size) + self.coverage_target = 0 if coverage_target is None else int(coverage_target) + if self.coverage_target not in [0, 1]: + logger.warn(f"Unsuppported coverage target request '{coverage_target}'. Using default coverage target of 1") + self.coverage_target = 1 + self.scenario_target = 0 if scenario_target is None else int(scenario_target) self._init_randomiser(seed) self._visualiser = self._init_visualiser(in_suite.name) if graph or export_graph_data else None + try: # a short trace without the need for repeating scenarios is preferred direct_tracestate = self._search_direct_trace() @@ -111,7 +119,7 @@ def process_test_suite(self, in_suite: Suite, *, seed: str | int | bytes | bytea self.tracestate = TraceState([s.src_id for s in self.scenarios]) self.tracestate.unreached = direct_tracestate.unreached logger.debug("Direct trace not discovered. Now exploring with loops, allowing repetition of scenarios.") - self._generate_next_batch(self.BATCH_SIZE) + self._generate_next_batch(self.batch_size) finally: # Draw the graph even when a timeout or user interrupt occurs if graph: self._write_visualisation(graph) @@ -126,7 +134,8 @@ def process_test_suite(self, in_suite: Suite, *, seed: str | int | bytes | bytea def next_scenario_request(self): try: if len(self.tracestate) <= self.index: - self._generate_next_batch(self.BATCH_SIZE) + self._generate_next_batch(self.batch_size) + logger.warn(f"Extending run with {self.batch_size} scenarios. now {len(self.tracestate)} scenarios long.") if len(self.tracestate) > self.index: self.out_suite.scenarios.append(self.tracestate[self.index].scenario) self.index += 1 @@ -284,7 +293,7 @@ def _generate_next_batch(self, batchsize): old_len = len(tracestate) self._update_visualisation(tracestate) while (len(tracestate) < old_len + batchsize) and \ - (not tracestate.coverage_reached() or (self.SCENARIO_TARGET and len(tracestate) < self.SCENARIO_TARGET)): + (not tracestate.coverage_reached() or (self.scenario_target and len(tracestate) < self.scenario_target)): candidate_id = tracestate.next_candidate(retry=True, randomise=True) if candidate_id is None: # No more candidates remaining for this level if not tracestate.can_rewind(): From 5c2a1331f4ba047b405233c89be4905e3a303336 Mon Sep 17 00:00:00 2001 From: JFoederer <32476108+JFoederer@users.noreply.github.com> Date: Sun, 24 May 2026 16:46:46 +0200 Subject: [PATCH 03/19] hit both coverage and scenario target --- robotmbt/modeller.py | 2 +- robotmbt/suiteprocessors.py | 9 +++++---- robotmbt/tracestate.py | 8 +++++--- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/robotmbt/modeller.py b/robotmbt/modeller.py index 0d74200..705f713 100644 --- a/robotmbt/modeller.py +++ b/robotmbt/modeller.py @@ -260,6 +260,6 @@ def rewind(tracestate: TraceState, drought_recovery: bool = False) -> TraceSnapS # When rewinding an 'in between' part, rewind both the part and the refinement tracestate.rewind() tail = tracestate.rewind() - while drought_recovery and tracestate.coverage_drought: + while drought_recovery and tracestate.coverage_drought and tracestate.can_rewind(): tail = tracestate.rewind() return tail diff --git a/robotmbt/suiteprocessors.py b/robotmbt/suiteprocessors.py index f9b2421..3980ca2 100644 --- a/robotmbt/suiteprocessors.py +++ b/robotmbt/suiteprocessors.py @@ -139,7 +139,7 @@ def next_scenario_request(self): if len(self.tracestate) > self.index: self.out_suite.scenarios.append(self.tracestate[self.index].scenario) self.index += 1 - self.tracestate.no_rewind += 1 + self.tracestate.rewind_limit += 1 except AttributeError: pass @@ -292,8 +292,9 @@ def _generate_next_batch(self, batchsize): tracestate = self.tracestate old_len = len(tracestate) self._update_visualisation(tracestate) - while (len(tracestate) < old_len + batchsize) and \ - (not tracestate.coverage_reached() or (self.scenario_target and len(tracestate) < self.scenario_target)): + while len(tracestate) < old_len + batchsize and \ + (self.coverage_target and not tracestate.coverage_reached() + or self.scenario_target and len(tracestate) < self.scenario_target): candidate_id = tracestate.next_candidate(retry=True, randomise=True) if candidate_id is None: # No more candidates remaining for this level if not tracestate.can_rewind(): @@ -319,7 +320,7 @@ def _generate_next_batch(self, batchsize): if self.__last_candidate_changed_nothing(tracestate): logger.debug("Repeated scenario did not change the model's state. Stop trying.") modeller.rewind(tracestate) - elif tracestate.coverage_drought > self.DROUGHT_LIMIT: + elif self.coverage_target and not self.tracestate.coverage_reached() and tracestate.coverage_drought > self.DROUGHT_LIMIT: logger.debug(f"Went too long without new coverage (>{self.DROUGHT_LIMIT}x). " "Roll back to last coverage increase and try something else.") modeller.rewind(tracestate, drought_recovery=True) diff --git a/robotmbt/tracestate.py b/robotmbt/tracestate.py index 665d821..0babff4 100644 --- a/robotmbt/tracestate.py +++ b/robotmbt/tracestate.py @@ -59,7 +59,9 @@ def __init__(self, scenario_indexes: list[int]): self._tried: list[list[int]] = [[]] # Keeps track of the scenarios already tried at each step in the trace self._snapshots: list[TraceSnapShot] = [] # Keeps details for elements in trace self._open_refinements: list[int] = [] - self.no_rewind = 0 + # The rewind limit indicates a (soft) limit for scenarios that should not be rewound. E.g. because they were + # already scheduled for execution. It refers to the number of scenarios that should remain in the trace. + self.rewind_limit = 0 @property def model(self) -> ModelSpace | None: @@ -123,7 +125,7 @@ def copy(self): cp._tried = [triedlist[:] for triedlist in self._tried] cp._snapshots = self._snapshots[:] cp._open_refinements = self._open_refinements[:] - cp.no_rewind = self.no_rewind + cp.rewind_limit = self.rewind_limit return cp def coverage_reached(self) -> bool: @@ -211,7 +213,7 @@ def push_partial_scenario(self, index: int, scenario: Scenario, model: ModelSpac self._snapshots.append(TraceSnapShot(id, scenario, model, remainder, self.coverage_drought)) def can_rewind(self) -> bool: - return len(self._snapshots[self.no_rewind:]) > 0 + return len(self._snapshots[self.rewind_limit:]) > 0 def rewind(self) -> TraceSnapShot | None: id = self._snapshots[-1].id From a1fed5ea7594f92e947831f1ffcaff61da82549a Mon Sep 17 00:00:00 2001 From: JFoederer <32476108+JFoederer@users.noreply.github.com> Date: Mon, 25 May 2026 14:02:43 +0200 Subject: [PATCH 04/19] handle rewinds when some scenarios are already executed --- robotmbt/modeller.py | 5 +- robotmbt/suiteprocessors.py | 6 +-- robotmbt/tracestate.py | 38 +++++++++++++- utest/test_tracestate.py | 80 +++++++++++++++++++++++------ utest/test_tracestate_refinement.py | 6 +-- 5 files changed, 108 insertions(+), 27 deletions(-) diff --git a/robotmbt/modeller.py b/robotmbt/modeller.py index 705f713..359a665 100644 --- a/robotmbt/modeller.py +++ b/robotmbt/modeller.py @@ -146,7 +146,7 @@ def handle_refinement_exit(inserted_refinement: Scenario, tracestate: TraceState tail_inserted, remainder, extra_data = process_scenario(refinement_tail, model) if not tail_inserted: logger.debug(extra_data['fail_msg']) - # Confirm then rewind, to roll back complete scenario, including its refiements + # Confirm then rewind, to roll back complete scenario, including its refinements # Because that exit check passed, this is an error in the refined scenario itself tracestate.confirm_full_scenario(refinement_tail.src_id, refinement_tail, model) tail = rewind(tracestate) @@ -256,9 +256,6 @@ def _parse_modifier_expression(expression: str, args: StepArguments) -> tuple[st def rewind(tracestate: TraceState, drought_recovery: bool = False) -> TraceSnapShot | None: - if tracestate[-1].remainder and tracestate.highest_part(tracestate[-1].remainder.src_id) > 1: - # When rewinding an 'in between' part, rewind both the part and the refinement - tracestate.rewind() tail = tracestate.rewind() while drought_recovery and tracestate.coverage_drought and tracestate.can_rewind(): tail = tracestate.rewind() diff --git a/robotmbt/suiteprocessors.py b/robotmbt/suiteprocessors.py index 3980ca2..4537cfd 100644 --- a/robotmbt/suiteprocessors.py +++ b/robotmbt/suiteprocessors.py @@ -135,7 +135,7 @@ def next_scenario_request(self): try: if len(self.tracestate) <= self.index: self._generate_next_batch(self.batch_size) - logger.warn(f"Extending run with {self.batch_size} scenarios. now {len(self.tracestate)} scenarios long.") + logger.warn(f"Extending run with max. {self.batch_size} scenarios. Now {len(self.tracestate)} long.") if len(self.tracestate) > self.index: self.out_suite.scenarios.append(self.tracestate[self.index].scenario) self.index += 1 @@ -293,8 +293,8 @@ def _generate_next_batch(self, batchsize): old_len = len(tracestate) self._update_visualisation(tracestate) while len(tracestate) < old_len + batchsize and \ - (self.coverage_target and not tracestate.coverage_reached() - or self.scenario_target and len(tracestate) < self.scenario_target): + (self.coverage_target and not tracestate.coverage_reached() + or self.scenario_target and len(tracestate) < self.scenario_target): candidate_id = tracestate.next_candidate(retry=True, randomise=True) if candidate_id is None: # No more candidates remaining for this level if not tracestate.can_rewind(): diff --git a/robotmbt/tracestate.py b/robotmbt/tracestate.py index 0babff4..f907764 100644 --- a/robotmbt/tracestate.py +++ b/robotmbt/tracestate.py @@ -213,13 +213,39 @@ def push_partial_scenario(self, index: int, scenario: Scenario, model: ModelSpac self._snapshots.append(TraceSnapShot(id, scenario, model, remainder, self.coverage_drought)) def can_rewind(self) -> bool: - return len(self._snapshots[self.rewind_limit:]) > 0 + rewind_margin = len(self._snapshots[self.rewind_limit:]) + if rewind_margin == 0: + return False + n = 1 + index, part = self.split_id(self._snapshots[-1].id) + if part and part > 1: + # When rewinding an 'in between' part, rewind both the part and the refinement + n += 1 + if part != 0: + return rewind_margin >= n + + # Refined scenarios that are already closed will be rewound in full. + # Check if the scenario's opening part is within the rewind margin. + for i in range(1, rewind_margin + 1): + if self._snapshots[-i].id == f'{index}.1': + n = i + return True + return False # went beyond rewind limit def rewind(self) -> TraceSnapShot | None: + """ + Performs a single rewind action, removing the most recently completed scenario from the trace. + + If the most recently completed scenario contained refinements, then the complete scenario, including + its refinements is rewound. If multi-part refinement is ongoing and a refinement step just ended, + without completing the full scenario, then the trace is rewound to before the latest refinement. + Use can_rewind() to check if a rewind is possible. + """ id = self._snapshots[-1].id - index = int(id.split('.')[0]) + index, part = self.split_id(id) self._snapshots.pop() if id.endswith('.0'): + # refined scenarios are rewinded in full self.c_pool[index] -= 1 self._open_refinements.append(index) while self._snapshots[-1].id != f"{index}.1": @@ -227,12 +253,20 @@ def rewind(self) -> TraceSnapShot | None: return self.rewind() self._tried.pop() + if part and part > 1: + # When rewinding an 'in between' part, rewind both the part and the refinement + return self.rewind() + if '.' not in id: self.c_pool[index] -= 1 if id.endswith('.1'): self._open_refinements.pop() return self._snapshots[-1] if self._snapshots else None + @staticmethod + def split_id(id: str) -> tuple[int, int | int, None]: + return tuple(map(int, id.split('.'))) if '.' in id else (int(id), None) + def __iter__(self): return iter(self._snapshots) diff --git a/utest/test_tracestate.py b/utest/test_tracestate.py index ed6c37a..f78b883 100644 --- a/utest/test_tracestate.py +++ b/utest/test_tracestate.py @@ -345,6 +345,15 @@ def test_rewind_includes_drought_update(self): ts.rewind() self.assertEqual(ts.coverage_drought, 0) + def test_rewind_limit(self): + ts = TraceState([1, 2]) + ts.confirm_full_scenario(1, ScenarioStub('one'), ModelStub()) + ts.confirm_full_scenario(2, ScenarioStub('two'), ModelStub()) + ts.rewind_limit = 1 + self.assertTrue(ts.can_rewind()) + ts.rewind() + self.assertFalse(ts.can_rewind()) + def test_trace_id_properties(self): ts = TraceState([4, 1, 2, 3]) ts.confirm_full_scenario(3, ScenarioStub(), ModelStub()) @@ -394,19 +403,20 @@ def test_rewind_of_single_part(self): self.assertEqual(ts.get_trace(), []) def test_rewind_all_parts(self): - ts = TraceState([1]) + ts = TraceState([1, 2]) ts.push_partial_scenario(1, ScenarioStub('part1'), ModelStub()) + ts.confirm_full_scenario(2, ScenarioStub('two'), ModelStub()) self.assertIs(ts.coverage_reached(), False) ts.push_partial_scenario(1, ScenarioStub('part2'), ModelStub()) self.assertIs(ts.coverage_reached(), False) - self.assertEqual(ts.get_trace(), ['part1', 'part2']) + self.assertEqual(ts.get_trace(), ['part1', 'two', 'part2']) self.assertIs(ts.next_candidate(), None) ts.rewind() self.assertEqual(ts.get_trace(), ['part1']) self.assertIs(ts.next_candidate(), None) ts.rewind() self.assertEqual(ts.get_trace(), []) - self.assertIs(ts.next_candidate(), None) + self.assertIs(ts.next_candidate(), 2) self.assertIs(ts.can_rewind(), False) def test_partial_scenario_still_excluded_from_candidacy_after_rewind(self): @@ -416,16 +426,16 @@ def test_partial_scenario_still_excluded_from_candidacy_after_rewind(self): ts.rewind() self.assertIs(ts.next_candidate(), None) - def test_rewind_to_partial_scenario(self): - ts = TraceState([1]) + def test_rewind_partial_to_partial_scenario(self): + ts = TraceState([1, 2]) ts.push_partial_scenario(1, ScenarioStub('part1'), ModelStub(a=1)) - ts.push_partial_scenario(1, ScenarioStub('part2'), ModelStub(b=2)) + ts.push_partial_scenario(2, ScenarioStub('part1'), ModelStub(b=2)) snapshot = ts.rewind() self.assertEqual(snapshot.id, '1.1') self.assertEqual(snapshot.scenario, 'part1') self.assertEqual(snapshot.model, dict(a=1)) - def test_rewind_last_part(self): + def test_rewind_partial_to_full_scenario(self): ts = TraceState([1, 2]) ts.confirm_full_scenario(1, ScenarioStub('one'), ModelStub(a=1)) ts.push_partial_scenario(2, ScenarioStub('part1'), ModelStub(b=2)) @@ -435,14 +445,25 @@ def test_rewind_last_part(self): self.assertEqual(snapshot.scenario, 'one') self.assertEqual(snapshot.model, dict(a=1)) - def test_rewind_all_parts_of_completed_scenario_at_once(self): - ts = TraceState([1]) + def test_rewind_full_to_partial_scenario(self): + ts = TraceState([1, 2]) ts.push_partial_scenario(1, ScenarioStub('part1'), ModelStub(a=1)) - ts.push_partial_scenario(1, ScenarioStub('part2'), ModelStub(b=2)) + ts.confirm_full_scenario(2, ScenarioStub('two'), ModelStub(b=2)) + snapshot = ts.rewind() + self.assertEqual(snapshot.id, '1.1') + self.assertEqual(snapshot.scenario, 'part1') + self.assertEqual(snapshot.model, dict(a=1)) + + def test_rewind_all_parts_of_completed_scenario_at_once(self): + ts = TraceState([1, 2, 3]) + ts.push_partial_scenario(1, ScenarioStub('part1'), ModelStub()) + ts.confirm_full_scenario(2, ScenarioStub('two'), ModelStub()) + ts.push_partial_scenario(1, ScenarioStub('part2'), ModelStub()) + ts.confirm_full_scenario(3, ScenarioStub('three'), ModelStub()) ts.confirm_full_scenario(1, ScenarioStub('remainder'), ModelStub()) tail = ts.rewind() self.assertEqual(ts.get_trace(), []) - self.assertIs(ts.next_candidate(), None) + self.assertIs(ts.next_candidate(), 2) self.assertIs(tail, None) def test_tried_entries_after_rewind(self): @@ -456,8 +477,6 @@ def test_tried_entries_after_rewind(self): ts.reject_scenario(21) self.assertEqual(ts.tried, [20, 21]) ts.rewind() - self.assertEqual(ts.tried, []) - ts.rewind() self.assertEqual(ts.tried, [10, 11, 2]) ts.reject_scenario(12) self.assertEqual(ts.tried, [10, 11, 2, 12]) @@ -486,8 +505,9 @@ def test_highest_part_after_completing_multiple_parts(self): self.assertEqual(ts.highest_part(1), 0) def test_highest_part_after_partial_rewind(self): - ts = TraceState([1]) + ts = TraceState([1, 2]) ts.push_partial_scenario(1, ScenarioStub('part1'), ModelStub()) + ts.confirm_full_scenario(2, ScenarioStub('two'), ModelStub()) ts.push_partial_scenario(1, ScenarioStub('part2'), ModelStub()) self.assertEqual(ts.highest_part(1), 2) ts.rewind() @@ -496,10 +516,12 @@ def test_highest_part_after_partial_rewind(self): self.assertEqual(ts.highest_part(1), 0) def test_highest_part_is_0_when_no_refinement_is_ongoing(self): - ts = TraceState([1]) + ts = TraceState([1, 2, 3]) self.assertEqual(ts.highest_part(1), 0) ts.push_partial_scenario(1, ScenarioStub('part1'), ModelStub()) + ts.confirm_full_scenario(2, ScenarioStub('two'), ModelStub()) ts.push_partial_scenario(1, ScenarioStub('part2'), ModelStub()) + ts.confirm_full_scenario(3, ScenarioStub('three'), ModelStub()) ts.confirm_full_scenario(1, ScenarioStub('remainder'), ModelStub()) self.assertEqual(ts.highest_part(1), 0) ts.rewind() @@ -597,6 +619,34 @@ def test_trace_id_properties_for_partials(self): self.assertEqual(ts.not_in_trace, [1, 2, 5, 7]) self.assertEqual(ts.unreached, [5, 7]) + def test_rewind_limit_for_partials(self): + ts = TraceState([1, 2, 3]) + ts.push_partial_scenario(1, ScenarioStub(), ModelStub()) + ts.confirm_full_scenario(2, ScenarioStub(), ModelStub()) + self.assertTrue(ts.can_rewind()) + ts.rewind_limit = 2 + self.assertFalse(ts.can_rewind()) + + ts.push_partial_scenario(1, ScenarioStub(), ModelStub()) + self.assertFalse(ts.can_rewind()) + + ts.confirm_full_scenario(3, ScenarioStub(), ModelStub()) + self.assertTrue(ts.can_rewind()) + ts.confirm_full_scenario(1, ScenarioStub(), ModelStub()) + self.assertFalse(ts.can_rewind()) + + def test_rewind_limit_just_before_partial(self): + ts = TraceState([1, 2, 3]) + ts.confirm_full_scenario(1, 'one', ModelStub()) + ts.rewind_limit = 1 + ts.push_partial_scenario(2, 'part1', ModelStub()) + self.assertTrue(ts.can_rewind()) + ts.confirm_full_scenario(3, 'three', ModelStub()) + ts.confirm_full_scenario(2, 'final part', ModelStub()) + self.assertTrue(ts.can_rewind()) + ts.rewind_limit = 2 + self.assertFalse(ts.can_rewind()) + class ScenarioStub(str): """Stub for suitedata.Scenario""" diff --git a/utest/test_tracestate_refinement.py b/utest/test_tracestate_refinement.py index 49440dc..1aa43da 100644 --- a/utest/test_tracestate_refinement.py +++ b/utest/test_tracestate_refinement.py @@ -97,7 +97,6 @@ def test_rewind_to_swap_refinements(self): inner2 = ts.next_candidate() ts.reject_scenario(inner2) ts.rewind() - ts.rewind() self.assertEqual(ts.tried, [inner1]) self.assertEqual(ts.next_candidate(), inner2) ts.confirm_full_scenario(inner2, 'B2', {}) @@ -123,7 +122,6 @@ def test_rewind_partial_scenario_to_before_outer(self): inner2 = ts.next_candidate() ts.reject_scenario(inner2) ts.rewind() - ts.rewind() previous = ts.rewind() self.assertEqual(previous.scenario, 'HEAD') self.assertEqual(ts.get_trace(), ['HEAD']) @@ -394,15 +392,17 @@ def test_is_refinement_active_by_index(self): self.assertFalse(ts.is_refinement_active(2)) def test_remainder_can_be_set_and_retrieved(self): - ts = TraceState([1, 2]) + ts = TraceState([1, 2, 3]) ts.push_partial_scenario(1, 'one part1', {}, 'one part2') ts.push_partial_scenario(2, 'two part1', {}, 'two parts 2+3') self.assertEqual(ts.get_remainder(1), 'one part2') self.assertEqual(ts.get_remainder(2), 'two parts 2+3') + ts.confirm_full_scenario(3, 'three', {}) ts.push_partial_scenario(2, 'two part2', {}, 'two part3') self.assertEqual(ts.get_remainder(2), 'two part3') ts.rewind() self.assertEqual(ts.get_remainder(2), 'two parts 2+3') + ts.confirm_full_scenario(3, 'three', {}) ts.push_partial_scenario(2, 'two part2', {}, 'two part3B') self.assertEqual(ts.get_remainder(2), 'two part3B') ts.confirm_full_scenario(2, 'two', {}) From d19ada8875bdebf2b078bb519c869fb4ef747b47 Mon Sep 17 00:00:00 2001 From: JFoederer <32476108+JFoederer@users.noreply.github.com> Date: Wed, 3 Jun 2026 13:38:12 +0200 Subject: [PATCH 05/19] refactor SuiteProcessor overrides --- robotmbt/__init__.py | 1 + robotmbt/suiteprocessors.py | 42 +++++++++++++++++++------------- robotmbt/suitereplacer.py | 46 +++++++++++++++-------------------- utest/test_suiteprocessors.py | 26 ++++++++++---------- 4 files changed, 59 insertions(+), 56 deletions(-) diff --git a/robotmbt/__init__.py b/robotmbt/__init__.py index bcdfcc7..236550e 100644 --- a/robotmbt/__init__.py +++ b/robotmbt/__init__.py @@ -32,6 +32,7 @@ from .version import VERSION from .suitereplacer import SuiteReplacer +from .suiteprocessors import SuiteProcessor class robotmbt(SuiteReplacer): diff --git a/robotmbt/suiteprocessors.py b/robotmbt/suiteprocessors.py index 4537cfd..ab09c08 100644 --- a/robotmbt/suiteprocessors.py +++ b/robotmbt/suiteprocessors.py @@ -47,12 +47,21 @@ Visualiser = None -class SuiteProcessors: - @staticmethod - def echo(in_suite: Suite) -> Suite: +class SuiteProcessor: + def process_test_suite(self, in_suite: Suite, **kwargs) -> Suite: + raise NotImplementedError() + + def next_scenario_request(self): + pass + + +class Echo(SuiteProcessor): + def process_test_suite(self, in_suite: Suite) -> Suite: return in_suite - def flatten(self, in_suite: Suite) -> Suite: + +class Flatten(SuiteProcessor): + def process_test_suite(self, in_suite: Suite) -> Suite: """ Takes a Suite as input and returns a Suite as output. The output Suite does not have any sub-suites, only scenarios. The scenarios do not have a setup. Any setup @@ -69,7 +78,7 @@ def flatten(self, in_suite: Suite) -> Suite: scenario.teardown = None out_suite.scenarios = [] for suite in in_suite.suites: - subsuite = self.flatten(suite) + subsuite = self.process_test_suite(suite) for scenario in subsuite.scenarios: if subsuite.setup: scenario.steps.insert(0, subsuite.setup) @@ -80,6 +89,8 @@ def flatten(self, in_suite: Suite) -> Suite: out_suite.suites = [] return out_suite + +class ModelBased(SuiteProcessor): def process_test_suite(self, in_suite: Suite, *, seed: str | int | bytes | bytearray = 'new', batch_size: str | int = 100, coverage_target: str | int | None = 1, @@ -89,7 +100,7 @@ def process_test_suite(self, in_suite: Suite, *, seed: str | int | bytes | bytea self.out_suite.filename = in_suite.filename self.out_suite.parent = in_suite.parent self._fail_on_step_errors(in_suite) - self.flat_suite = self.flatten(in_suite) + self.flat_suite = Flatten().process_test_suite(in_suite) for id, scenario in enumerate(self.flat_suite.scenarios, start=1): scenario.src_id = id self.scenarios: list[Scenario] = self.flat_suite.scenarios[:] @@ -132,16 +143,13 @@ def process_test_suite(self, in_suite: Suite, *, seed: str | int | bytes | bytea return self.out_suite def next_scenario_request(self): - try: - if len(self.tracestate) <= self.index: - self._generate_next_batch(self.batch_size) - logger.warn(f"Extending run with max. {self.batch_size} scenarios. Now {len(self.tracestate)} long.") - if len(self.tracestate) > self.index: - self.out_suite.scenarios.append(self.tracestate[self.index].scenario) - self.index += 1 - self.tracestate.rewind_limit += 1 - except AttributeError: - pass + if len(self.tracestate) <= self.index: + self._generate_next_batch(self.batch_size) + logger.warn(f"Extending run with max. {self.batch_size} scenarios. Now {len(self.tracestate)} long.") + if len(self.tracestate) > self.index: + self.out_suite.scenarios.append(self.tracestate[self.index].scenario) + self.index += 1 + self.tracestate.rewind_limit += 1 def draw_graph_from_export_file(self, file_path: str, graph_style: str): self._visualiser = self._init_visualiser() @@ -387,7 +395,7 @@ def _init_randomiser(seed: str | int | bytes | bytearray | None): "Using system's random seed for trace generation. This trace cannot be rerun. Use `seed=new` to generate a reusable seed.") elif str(seed).lower() == 'new': random.seed() - new_seed = SuiteProcessors._generate_seed() + new_seed = ModelBased._generate_seed() logger.info(f"seed={new_seed} (use seed to rerun this trace)") random.seed(new_seed) else: diff --git a/robotmbt/suitereplacer.py b/robotmbt/suitereplacer.py index ab7b878..9e2c443 100644 --- a/robotmbt/suitereplacer.py +++ b/robotmbt/suitereplacer.py @@ -40,39 +40,32 @@ from robot.libraries.BuiltIn import BuiltIn from .suitedata import Suite, Scenario, Step -from .suiteprocessors import SuiteProcessors +from .suiteprocessors import SuiteProcessor, ModelBased, Echo, Flatten Robot = BuiltIn() @library(scope="GLOBAL", listener='SELF') class SuiteReplacer: - def __init__(self, processor: str = 'process_test_suite', processor_lib: str | None = None): + def __init__(self, processor: str = 'robotmbt'): self.current_suite: robot.model.TestSuite | None = None self.mbt_anchor_suite: robot.model.TestSuite | None = None - self.processor_lib_name: str | None = processor_lib self.processor_name: str = processor - self._processor_lib: SuiteProcessors | None | object = None - self._processor_method: Callable[..., Suite] | None = None + self.processor: SuiteProcessor | None = None self.suite_gen: list[Iterator[Suite]] = [] # Generator for on-the-fly suite insertion self.test_case_gen: list[Iterator[Scenario]] = [] # Generator for on-the-fly test case insertion self.processor_options: dict[str, Any] = {} - @property - def processor_lib(self) -> SuiteProcessors: - if self._processor_lib is None: - self._processor_lib = SuiteProcessors() if self.processor_lib_name is None \ - else Robot.get_library_instance(self.processor_lib_name) - return self._processor_lib - - @property - def processor_method(self): - if self._processor_method is None: - if not hasattr(self.processor_lib, self.processor_name): - Robot.fail( - f"Processor '{self.processor_name}' not available for model-based processor library {self.processor_lib_name}") - self._processor_method = getattr(self._processor_lib, self.processor_name) - return self._processor_method + def load_processor(self): + if self.processor_name.lower() == 'robotmbt': + self.processor = ModelBased() + elif self.processor_name.lower() == 'echo': + self.processor = Echo() + elif self.processor_name.lower() == 'flatten': + self.processor = Flatten() + else: + self.processor = Robot.get_library_instance(self.processor_name) + return self.processor @keyword(name="Treat this test suite Model-based") def treat_model_based(self, **kwargs): @@ -93,7 +86,8 @@ def treat_model_based(self, **kwargs): local_settings = self.processor_options.copy() local_settings.update(kwargs) master_suite = self.__process_robot_suite(self.current_suite, parent=None) - modelbased_suite = self.processor_method(master_suite, **local_settings) + self.load_processor() + modelbased_suite = self.processor.process_test_suite(master_suite, **local_settings) self.suite_gen = [iter(modelbased_suite.suites)] self.test_case_gen = [iter(modelbased_suite.scenarios)] self.__clearTestSuite(self.current_suite) @@ -125,7 +119,7 @@ def show_graph(self, json_file_path: str, graph_style: str = 'scenario'): different graph style than was used during the test run. If no graph style is selected, then the scenario graph style is used. """ - SuiteProcessors().draw_graph_from_export_file(json_file_path, graph_style) + ModelBased().draw_graph_from_export_file(json_file_path, graph_style) def __process_robot_suite(self, in_suite: robot.model.TestSuite, parent: Suite | None) -> Suite: out_suite = Suite(in_suite.name, parent) @@ -185,8 +179,7 @@ def add_next_new(self, target_suite: robot.model.TestSuite): new_target = self.add_suite(new_suite, target_suite) self.add_next_new(new_target) except StopIteration: - if hasattr(self.processor_lib, 'next_scenario_request'): - self.processor_lib.next_scenario_request() + self.processor.next_scenario_request() try: self.add_test(next(self.test_case_gen[-1]), target_suite) except StopIteration: @@ -240,8 +233,9 @@ def _end_suite(self, suite: robot.model.TestSuite, result): def _end_test(self, test_case: robot.model.TestCase, result): if not self.mbt_anchor_suite: return - if hasattr(self.processor_lib, 'next_scenario_request'): - self.processor_lib.next_scenario_request() + if not isinstance(self.processor, SuiteProcessor): + raise TypeError("processor must be of type SuiteProcessor") + self.processor.next_scenario_request() try: self.add_test(next(self.test_case_gen[-1]), self.current_suite) except StopIteration: diff --git a/utest/test_suiteprocessors.py b/utest/test_suiteprocessors.py index 1c8970a..e616049 100644 --- a/utest/test_suiteprocessors.py +++ b/utest/test_suiteprocessors.py @@ -33,50 +33,50 @@ import unittest from unittest.mock import patch, call -from robotmbt.suiteprocessors import SuiteProcessors +from robotmbt.suiteprocessors import ModelBased @patch('robotmbt.suiteprocessors.random.seed') class TestRandomSeeding(unittest.TestCase): def test_provided_seed_is_used_as_is(self, mock): - SuiteProcessors._init_randomiser("specific seed") + ModelBased._init_randomiser("specific seed") mock.assert_called_with("specific seed") def test_provided_seed_is_stripped(self, mock): - SuiteProcessors._init_randomiser(" specific seed\t") + ModelBased._init_randomiser(" specific seed\t") mock.assert_called_with("specific seed") def test_seed_none_keeps_system_seed(self, mock): - SuiteProcessors._init_randomiser(None) + ModelBased._init_randomiser(None) mock.assert_called_with() def test_seed_none_as_string(self, mock): - SuiteProcessors._init_randomiser("None") + ModelBased._init_randomiser("None") mock.assert_called_with() def test_seed_none_as_string_is_stripped(self, mock): - SuiteProcessors._init_randomiser(" None\t") + ModelBased._init_randomiser(" None\t") mock.assert_called_with() def test_seed_none_as_string_is_case_insensitive(self, mock): - SuiteProcessors._init_randomiser("nOnE") + ModelBased._init_randomiser("nOnE") mock.assert_called_with() def test_seed_new_generates_reusable_seed(self, mock): - SuiteProcessors._init_randomiser("new") + ModelBased._init_randomiser("new") self._is_generated_seed(mock.call_args.args[0]) def test_seed_new_is_stripped(self, mock): - SuiteProcessors._init_randomiser(" new\t") + ModelBased._init_randomiser(" new\t") self._is_generated_seed(mock.call_args.args[0]) def test_seed_new_is_case_insensitive(self, mock): - SuiteProcessors._init_randomiser("NeW") + ModelBased._init_randomiser("NeW") self._is_generated_seed(mock.call_args.args[0]) def test_generated_seeds_have_max_2_consecutive_vowels_or_consonants(self, mock): for _ in range(20): - SuiteProcessors._init_randomiser("new") + ModelBased._init_randomiser("new") new_seed = mock.call_args.args[0] self._is_generated_seed(new_seed) self.assertNotIn('***', new_seed.translate({ord(c): '*' for c in 'aeiouy'})) @@ -87,8 +87,8 @@ def test_seed_is_reset_after_using_specific_seed(self, mock): added to cover the issue where, after having rerun a specific trace, the next generated seed was always the same. """ - SuiteProcessors._init_randomiser("specific seed") - SuiteProcessors._init_randomiser("new") + ModelBased._init_randomiser("specific seed") + ModelBased._init_randomiser("new") new_seed = mock.call_args.args[0] mock.assert_has_calls([call("specific seed"), call(), call(new_seed)]) From d92f852f9d62b590516aa5baa4b0f38a51adfa09 Mon Sep 17 00:00:00 2001 From: JFoederer <32476108+JFoederer@users.noreply.github.com> Date: Wed, 3 Jun 2026 13:45:12 +0200 Subject: [PATCH 06/19] use new-style SuiteProcessor --- .../03__parse_model_info/MyProcessor.py | 4 +++- .../03__parse_model_info/correct_model_info.robot | 2 +- .../03__parse_model_info/incorrect_model_info.robot | 2 +- .../option_handling/01__pass_option_directly.robot | 2 +- .../option_handling/02__set_option_by_keyword.robot | 2 +- .../03__update_option_by_keyword.robot | 2 +- .../04__update_option_at_trigger.robot | 2 +- .../05__use_update_without_setter.robot | 2 +- .../option_handling/06__pass_multiple_options.robot | 2 +- .../07__set_clears_other_options.robot | 2 +- .../08__empty_setter_clears_all_options.robot | 2 +- .../09__multiple_options_from_dict.robot | 2 +- .../option_handling/10__partial_option_update.robot | 2 +- .../option_handling/11__argument_restrictions.robot | 8 ++++---- .../__init__.robot | 2 +- .../with_bonus_scenario.robot | 3 ++- .../without_bonus_scenario.robot | 3 ++- .../01__with_bonus_scenario_option.robot | 3 ++- .../02__without_using_bonus_scenario_option.robot | 3 ++- .../__init__.robot | 2 +- ...__direct_setting_overrules_library_setting.robot | 3 ++- .../04__prior_overrule_does_not_persist.robot | 3 ++- .../__init__.robot | 2 +- .../option_handling/strictsuiterepeater.py | 13 +++++++++++++ .../option_handling/suiterepeater.py | 7 ++----- 25 files changed, 49 insertions(+), 31 deletions(-) create mode 100644 atest/robotMBT tests/07__processor_options/option_handling/strictsuiterepeater.py diff --git a/atest/robotMBT tests/03__parse_model_info/MyProcessor.py b/atest/robotMBT tests/03__parse_model_info/MyProcessor.py index 09cb13b..1d56771 100644 --- a/atest/robotMBT tests/03__parse_model_info/MyProcessor.py +++ b/atest/robotMBT tests/03__parse_model_info/MyProcessor.py @@ -1,4 +1,6 @@ -class MyProcessor: +from robotmbt import SuiteProcessor + +class MyProcessor(SuiteProcessor): def process_test_suite(self, in_suite): self.in_suite = in_suite diff --git a/atest/robotMBT tests/03__parse_model_info/correct_model_info.robot b/atest/robotMBT tests/03__parse_model_info/correct_model_info.robot index bbab3e8..1d6cd4e 100644 --- a/atest/robotMBT tests/03__parse_model_info/correct_model_info.robot +++ b/atest/robotMBT tests/03__parse_model_info/correct_model_info.robot @@ -1,7 +1,7 @@ *** Settings *** Suite Setup Treat this test suite Model-based Library MyProcessor.py -Library robotmbt processor_lib=MyProcessor +Library robotmbt processor=MyProcessor *** Test cases *** concise model info diff --git a/atest/robotMBT tests/03__parse_model_info/incorrect_model_info.robot b/atest/robotMBT tests/03__parse_model_info/incorrect_model_info.robot index 34c9fb2..a39197f 100644 --- a/atest/robotMBT tests/03__parse_model_info/incorrect_model_info.robot +++ b/atest/robotMBT tests/03__parse_model_info/incorrect_model_info.robot @@ -1,7 +1,7 @@ *** Settings *** Suite Setup Expect failing suite processing Library MyProcessor.py -Library robotmbt processor_lib=MyProcessor +Library robotmbt processor=MyProcessor *** Test Cases *** fail on empty model info diff --git a/atest/robotMBT tests/07__processor_options/option_handling/01__pass_option_directly.robot b/atest/robotMBT tests/07__processor_options/option_handling/01__pass_option_directly.robot index d482083..21817a1 100644 --- a/atest/robotMBT tests/07__processor_options/option_handling/01__pass_option_directly.robot +++ b/atest/robotMBT tests/07__processor_options/option_handling/01__pass_option_directly.robot @@ -3,7 +3,7 @@ Suite Setup Run keywords Set suite variable ${test_count} ${0} ... AND Treat this test suite Model-based repeat=2 Suite Teardown Should be equal ${test_count} ${2} Library suiterepeater.py -Library robotmbt processor_lib=suiterepeater +Library robotmbt processor=suiterepeater *** Test Cases *** only test case diff --git a/atest/robotMBT tests/07__processor_options/option_handling/02__set_option_by_keyword.robot b/atest/robotMBT tests/07__processor_options/option_handling/02__set_option_by_keyword.robot index bf673c8..8015bdb 100644 --- a/atest/robotMBT tests/07__processor_options/option_handling/02__set_option_by_keyword.robot +++ b/atest/robotMBT tests/07__processor_options/option_handling/02__set_option_by_keyword.robot @@ -4,7 +4,7 @@ Suite Setup Run keywords Set suite variable ${test_count} ${0} ... AND Treat this test suite Model-based Suite Teardown Should be equal ${test_count} ${2} Library suiterepeater.py -Library robotmbt processor_lib=suiterepeater +Library robotmbt processor=suiterepeater *** Test Cases *** only test case diff --git a/atest/robotMBT tests/07__processor_options/option_handling/03__update_option_by_keyword.robot b/atest/robotMBT tests/07__processor_options/option_handling/03__update_option_by_keyword.robot index f95ef80..388fc66 100644 --- a/atest/robotMBT tests/07__processor_options/option_handling/03__update_option_by_keyword.robot +++ b/atest/robotMBT tests/07__processor_options/option_handling/03__update_option_by_keyword.robot @@ -5,7 +5,7 @@ Suite Setup Run keywords Set suite variable ${test_count} ${0} ... AND Treat this test suite Model-based Suite Teardown Should be equal ${test_count} ${3} Library suiterepeater.py -Library robotmbt processor_lib=suiterepeater +Library robotmbt processor=suiterepeater *** Test Cases *** only test case diff --git a/atest/robotMBT tests/07__processor_options/option_handling/04__update_option_at_trigger.robot b/atest/robotMBT tests/07__processor_options/option_handling/04__update_option_at_trigger.robot index 4d34f75..55289dc 100644 --- a/atest/robotMBT tests/07__processor_options/option_handling/04__update_option_at_trigger.robot +++ b/atest/robotMBT tests/07__processor_options/option_handling/04__update_option_at_trigger.robot @@ -4,7 +4,7 @@ Suite Setup Run keywords Set suite variable ${test_count} ${0} ... AND Treat this test suite Model-based repeat=3 Suite Teardown Should be equal ${test_count} ${3} Library suiterepeater.py -Library robotmbt processor_lib=suiterepeater +Library robotmbt processor=suiterepeater *** Test Cases *** only test case diff --git a/atest/robotMBT tests/07__processor_options/option_handling/05__use_update_without_setter.robot b/atest/robotMBT tests/07__processor_options/option_handling/05__use_update_without_setter.robot index b32f658..cdcd3bc 100644 --- a/atest/robotMBT tests/07__processor_options/option_handling/05__use_update_without_setter.robot +++ b/atest/robotMBT tests/07__processor_options/option_handling/05__use_update_without_setter.robot @@ -4,7 +4,7 @@ Suite Setup Run keywords Set suite variable ${test_count} ${0} ... AND Treat this test suite Model-based Suite Teardown Should be equal ${test_count} ${2} Library suiterepeater.py -Library robotmbt processor_lib=suiterepeater +Library robotmbt processor=suiterepeater *** Test Cases *** only test case diff --git a/atest/robotMBT tests/07__processor_options/option_handling/06__pass_multiple_options.robot b/atest/robotMBT tests/07__processor_options/option_handling/06__pass_multiple_options.robot index 530739c..c589ec2 100644 --- a/atest/robotMBT tests/07__processor_options/option_handling/06__pass_multiple_options.robot +++ b/atest/robotMBT tests/07__processor_options/option_handling/06__pass_multiple_options.robot @@ -3,7 +3,7 @@ Suite Setup Run keywords Set suite variable ${test_count} ${0} ... AND Treat this test suite Model-based repeat=2 bonus_scenario=${True} Suite Teardown Should be equal ${test_count} ${3} Library suiterepeater.py -Library robotmbt processor_lib=suiterepeater +Library robotmbt processor=suiterepeater *** Test Cases *** only test case diff --git a/atest/robotMBT tests/07__processor_options/option_handling/07__set_clears_other_options.robot b/atest/robotMBT tests/07__processor_options/option_handling/07__set_clears_other_options.robot index 0f9ff6c..6d472f0 100644 --- a/atest/robotMBT tests/07__processor_options/option_handling/07__set_clears_other_options.robot +++ b/atest/robotMBT tests/07__processor_options/option_handling/07__set_clears_other_options.robot @@ -5,7 +5,7 @@ Suite Setup Run keywords Set suite variable ${test_count} ${0} ... AND Treat this test suite Model-based Suite Teardown Should be equal ${test_count} ${2} Library suiterepeater.py -Library robotmbt processor_lib=suiterepeater +Library robotmbt processor=suiterepeater *** Test Cases *** only test case diff --git a/atest/robotMBT tests/07__processor_options/option_handling/08__empty_setter_clears_all_options.robot b/atest/robotMBT tests/07__processor_options/option_handling/08__empty_setter_clears_all_options.robot index 292ee34..6b7fe5b 100644 --- a/atest/robotMBT tests/07__processor_options/option_handling/08__empty_setter_clears_all_options.robot +++ b/atest/robotMBT tests/07__processor_options/option_handling/08__empty_setter_clears_all_options.robot @@ -5,7 +5,7 @@ Suite Setup Run keywords Set suite variable ${test_count} ${0} ... AND Treat this test suite Model-based Suite Teardown Should be equal ${test_count} ${1} Library suiterepeater.py -Library robotmbt processor_lib=suiterepeater +Library robotmbt processor=suiterepeater *** Test Cases *** only test case diff --git a/atest/robotMBT tests/07__processor_options/option_handling/09__multiple_options_from_dict.robot b/atest/robotMBT tests/07__processor_options/option_handling/09__multiple_options_from_dict.robot index 1d0443c..f7ee4f3 100644 --- a/atest/robotMBT tests/07__processor_options/option_handling/09__multiple_options_from_dict.robot +++ b/atest/robotMBT tests/07__processor_options/option_handling/09__multiple_options_from_dict.robot @@ -3,7 +3,7 @@ Suite Setup Run keywords Set suite variable ${test_count} ${0} ... AND Treat this test suite Model-based &{mbt_options} Suite Teardown Should be equal ${test_count} ${3} Library suiterepeater.py -Library robotmbt processor_lib=suiterepeater +Library robotmbt processor=suiterepeater *** Variables *** &{mbt_options} repeat=2 bonus_scenario=${True} diff --git a/atest/robotMBT tests/07__processor_options/option_handling/10__partial_option_update.robot b/atest/robotMBT tests/07__processor_options/option_handling/10__partial_option_update.robot index e840d71..80a88dd 100644 --- a/atest/robotMBT tests/07__processor_options/option_handling/10__partial_option_update.robot +++ b/atest/robotMBT tests/07__processor_options/option_handling/10__partial_option_update.robot @@ -4,7 +4,7 @@ Suite Setup Run keywords Set suite variable ${test_count} ${0} ... AND Treat this test suite Model-based bonus_scenario=${False} Suite Teardown Should be equal ${test_count} ${2} Library suiterepeater.py -Library robotmbt processor_lib=suiterepeater +Library robotmbt processor=suiterepeater *** Variables *** &{mbt_options} repeat=2 bonus_scenario=${True} diff --git a/atest/robotMBT tests/07__processor_options/option_handling/11__argument_restrictions.robot b/atest/robotMBT tests/07__processor_options/option_handling/11__argument_restrictions.robot index 3da3a8f..e6fc9f6 100644 --- a/atest/robotMBT tests/07__processor_options/option_handling/11__argument_restrictions.robot +++ b/atest/robotMBT tests/07__processor_options/option_handling/11__argument_restrictions.robot @@ -1,12 +1,12 @@ *** Settings *** -Library suiterepeater.py -Library robotmbt processor_lib=suiterepeater processor=mandatory_repeat_argument +Library strictsuiterepeater.py +Library robotmbt processor=strictsuiterepeater *** Test Cases *** arguments can be mandatory - Run keyword and expect error *SuiteRepeater.mandatory_repeat_argument() missing 1 required keyword-only argument: 'repeat' + Run keyword and expect error *StrictSuiteRepeater.process_test_suite() missing 1 required keyword-only argument: 'repeat' ... Treat this test suite Model-based bonus_scenario=${True} can fail on unknown arguments - Run keyword and expect error *SuiteRepeater.mandatory_repeat_argument() got an unexpected keyword argument 'intentional_fail' + Run keyword and expect error *StrictSuiteRepeater.process_test_suite() got an unexpected keyword argument 'intentional_fail' ... Treat this test suite Model-based repeat=1 bonus_scenario=${True} intentional_fail=${True} diff --git a/atest/robotMBT tests/07__processor_options/option_handling/12__settings_can_span_multiple_suites/__init__.robot b/atest/robotMBT tests/07__processor_options/option_handling/12__settings_can_span_multiple_suites/__init__.robot index 37052f9..43ef370 100644 --- a/atest/robotMBT tests/07__processor_options/option_handling/12__settings_can_span_multiple_suites/__init__.robot +++ b/atest/robotMBT tests/07__processor_options/option_handling/12__settings_can_span_multiple_suites/__init__.robot @@ -3,4 +3,4 @@ Documentation In this suite one of the processor options is set on the highe ... which is then reused in both sub suites. Each sub suite adds their own value ... for a second configuration option. Suite Setup Set model-based options repeat=2 -Library robotmbt processor_lib=suiterepeater +Library robotmbt processor=suiterepeater diff --git a/atest/robotMBT tests/07__processor_options/option_handling/12__settings_can_span_multiple_suites/with_bonus_scenario.robot b/atest/robotMBT tests/07__processor_options/option_handling/12__settings_can_span_multiple_suites/with_bonus_scenario.robot index b7289d5..cb71d77 100644 --- a/atest/robotMBT tests/07__processor_options/option_handling/12__settings_can_span_multiple_suites/with_bonus_scenario.robot +++ b/atest/robotMBT tests/07__processor_options/option_handling/12__settings_can_span_multiple_suites/with_bonus_scenario.robot @@ -2,7 +2,8 @@ Suite Setup Run keywords Set suite variable ${test_count} ${0} ... AND Treat this test suite Model-based bonus_scenario=${True} Suite Teardown Should be equal ${test_count} ${3} -Library robotmbt processor_lib=suiterepeater +Library ../suiterepeater.py +Library robotmbt processor=suiterepeater *** Test Cases *** only test case diff --git a/atest/robotMBT tests/07__processor_options/option_handling/12__settings_can_span_multiple_suites/without_bonus_scenario.robot b/atest/robotMBT tests/07__processor_options/option_handling/12__settings_can_span_multiple_suites/without_bonus_scenario.robot index f3d6b99..73af861 100644 --- a/atest/robotMBT tests/07__processor_options/option_handling/12__settings_can_span_multiple_suites/without_bonus_scenario.robot +++ b/atest/robotMBT tests/07__processor_options/option_handling/12__settings_can_span_multiple_suites/without_bonus_scenario.robot @@ -2,7 +2,8 @@ Suite Setup Run keywords Set suite variable ${test_count} ${0} ... AND Treat this test suite Model-based bonus_scenario=${False} Suite Teardown Should be equal ${test_count} ${2} -Library robotmbt processor_lib=suiterepeater +Library ../suiterepeater.py +Library robotmbt processor=suiterepeater *** Test Cases *** only test case diff --git a/atest/robotMBT tests/07__processor_options/option_handling/13__direct_settings_affect_current_suite_only/01__with_bonus_scenario_option.robot b/atest/robotMBT tests/07__processor_options/option_handling/13__direct_settings_affect_current_suite_only/01__with_bonus_scenario_option.robot index b7289d5..cb71d77 100644 --- a/atest/robotMBT tests/07__processor_options/option_handling/13__direct_settings_affect_current_suite_only/01__with_bonus_scenario_option.robot +++ b/atest/robotMBT tests/07__processor_options/option_handling/13__direct_settings_affect_current_suite_only/01__with_bonus_scenario_option.robot @@ -2,7 +2,8 @@ Suite Setup Run keywords Set suite variable ${test_count} ${0} ... AND Treat this test suite Model-based bonus_scenario=${True} Suite Teardown Should be equal ${test_count} ${3} -Library robotmbt processor_lib=suiterepeater +Library ../suiterepeater.py +Library robotmbt processor=suiterepeater *** Test Cases *** only test case diff --git a/atest/robotMBT tests/07__processor_options/option_handling/13__direct_settings_affect_current_suite_only/02__without_using_bonus_scenario_option.robot b/atest/robotMBT tests/07__processor_options/option_handling/13__direct_settings_affect_current_suite_only/02__without_using_bonus_scenario_option.robot index 70f501a..f06478b 100644 --- a/atest/robotMBT tests/07__processor_options/option_handling/13__direct_settings_affect_current_suite_only/02__without_using_bonus_scenario_option.robot +++ b/atest/robotMBT tests/07__processor_options/option_handling/13__direct_settings_affect_current_suite_only/02__without_using_bonus_scenario_option.robot @@ -2,7 +2,8 @@ Suite Setup Run keywords Set suite variable ${test_count} ${0} ... AND Treat this test suite Model-based Suite Teardown Should be equal ${test_count} ${2} -Library robotmbt processor_lib=suiterepeater +Library ../suiterepeater.py +Library robotmbt processor=suiterepeater *** Test Cases *** only test case diff --git a/atest/robotMBT tests/07__processor_options/option_handling/13__direct_settings_affect_current_suite_only/__init__.robot b/atest/robotMBT tests/07__processor_options/option_handling/13__direct_settings_affect_current_suite_only/__init__.robot index 882097c..eb7426d 100644 --- a/atest/robotMBT tests/07__processor_options/option_handling/13__direct_settings_affect_current_suite_only/__init__.robot +++ b/atest/robotMBT tests/07__processor_options/option_handling/13__direct_settings_affect_current_suite_only/__init__.robot @@ -5,4 +5,4 @@ Documentation In this suite one of the processor options is set on the highe ... all. The second suite should be unaffected by the option set in the preceeding ... suite. Suite Setup Set model-based options repeat=2 -Library robotmbt processor_lib=suiterepeater +Library robotmbt processor=suiterepeater diff --git a/atest/robotMBT tests/07__processor_options/option_handling/14__overruling_library_setting_affects_current_suite_only/03__direct_setting_overrules_library_setting.robot b/atest/robotMBT tests/07__processor_options/option_handling/14__overruling_library_setting_affects_current_suite_only/03__direct_setting_overrules_library_setting.robot index f92713d..db5ce71 100644 --- a/atest/robotMBT tests/07__processor_options/option_handling/14__overruling_library_setting_affects_current_suite_only/03__direct_setting_overrules_library_setting.robot +++ b/atest/robotMBT tests/07__processor_options/option_handling/14__overruling_library_setting_affects_current_suite_only/03__direct_setting_overrules_library_setting.robot @@ -2,7 +2,8 @@ Suite Setup Run keywords Set suite variable ${test_count} ${0} ... AND Treat this test suite Model-based repeat=3 Suite Teardown Should be equal ${test_count} ${3} -Library robotmbt processor_lib=suiterepeater +Library ../suiterepeater.py +Library robotmbt processor=suiterepeater *** Test Cases *** only test case diff --git a/atest/robotMBT tests/07__processor_options/option_handling/14__overruling_library_setting_affects_current_suite_only/04__prior_overrule_does_not_persist.robot b/atest/robotMBT tests/07__processor_options/option_handling/14__overruling_library_setting_affects_current_suite_only/04__prior_overrule_does_not_persist.robot index 70f501a..f06478b 100644 --- a/atest/robotMBT tests/07__processor_options/option_handling/14__overruling_library_setting_affects_current_suite_only/04__prior_overrule_does_not_persist.robot +++ b/atest/robotMBT tests/07__processor_options/option_handling/14__overruling_library_setting_affects_current_suite_only/04__prior_overrule_does_not_persist.robot @@ -2,7 +2,8 @@ Suite Setup Run keywords Set suite variable ${test_count} ${0} ... AND Treat this test suite Model-based Suite Teardown Should be equal ${test_count} ${2} -Library robotmbt processor_lib=suiterepeater +Library ../suiterepeater.py +Library robotmbt processor=suiterepeater *** Test Cases *** only test case diff --git a/atest/robotMBT tests/07__processor_options/option_handling/14__overruling_library_setting_affects_current_suite_only/__init__.robot b/atest/robotMBT tests/07__processor_options/option_handling/14__overruling_library_setting_affects_current_suite_only/__init__.robot index 8ae64cf..a4b01e4 100644 --- a/atest/robotMBT tests/07__processor_options/option_handling/14__overruling_library_setting_affects_current_suite_only/__init__.robot +++ b/atest/robotMBT tests/07__processor_options/option_handling/14__overruling_library_setting_affects_current_suite_only/__init__.robot @@ -4,4 +4,4 @@ Documentation In this suite one of the processor options is set on the highe ... setting with their own value, the second library doesn't. The second suite should ... be unaffected by the overruled option from the preceeding suite. Suite Setup Set model-based options repeat=2 -Library robotmbt processor_lib=suiterepeater +Library robotmbt processor=suiterepeater diff --git a/atest/robotMBT tests/07__processor_options/option_handling/strictsuiterepeater.py b/atest/robotMBT tests/07__processor_options/option_handling/strictsuiterepeater.py new file mode 100644 index 0000000..fd38dd8 --- /dev/null +++ b/atest/robotMBT tests/07__processor_options/option_handling/strictsuiterepeater.py @@ -0,0 +1,13 @@ +from robot.api.deco import library + +from suiterepeater import SuiteRepeater + +@library(auto_keywords=None, listener=True) +class StrictSuiteRepeater(SuiteRepeater): + """ + Nearly identical to SuiteRepeater as used in other test cases. The difference is that + this variant is strict in its argument handling and will fail if mandatory arguments + are missing or unknown arguments are provided. + """ + def process_test_suite(self, in_suite, *, repeat, bonus_scenario=False): + return super().process_test_suite(in_suite, repeat=repeat, bonus_scenario=bonus_scenario) diff --git a/atest/robotMBT tests/07__processor_options/option_handling/suiterepeater.py b/atest/robotMBT tests/07__processor_options/option_handling/suiterepeater.py index bdbfa9f..1679f43 100644 --- a/atest/robotMBT tests/07__processor_options/option_handling/suiterepeater.py +++ b/atest/robotMBT tests/07__processor_options/option_handling/suiterepeater.py @@ -1,10 +1,10 @@ import copy from robot.api.deco import library - +from robotmbt import SuiteProcessor @library(auto_keywords=None, listener=True) -class SuiteRepeater: +class SuiteRepeater(SuiteProcessor): """ Given a test suite, repeats all scenarios 'repeat' times (default=1) Setting bonus_scenario=${True} repeats 1 additional time @@ -22,6 +22,3 @@ def process_test_suite(self, in_suite, repeat=1, **kwargs): if i: out_suite.scenarios[i].name += f" (rep {i+1})" return out_suite - - def mandatory_repeat_argument(self, in_suite, *, repeat, bonus_scenario=False): - return self.process_test_suite(in_suite, repeat=repeat, bonus_scenario=bonus_scenario) From 83eb6c59e7da16085b6f72a65e78c7523d127b71 Mon Sep 17 00:00:00 2001 From: JFoederer <32476108+JFoederer@users.noreply.github.com> Date: Wed, 3 Jun 2026 13:50:25 +0200 Subject: [PATCH 07/19] pep8 spacing --- atest/robotMBT tests/03__parse_model_info/MyProcessor.py | 2 +- .../option_handling/strictsuiterepeater.py | 2 ++ .../07__processor_options/option_handling/suiterepeater.py | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/atest/robotMBT tests/03__parse_model_info/MyProcessor.py b/atest/robotMBT tests/03__parse_model_info/MyProcessor.py index 1d56771..ac2d9a0 100644 --- a/atest/robotMBT tests/03__parse_model_info/MyProcessor.py +++ b/atest/robotMBT tests/03__parse_model_info/MyProcessor.py @@ -1,7 +1,7 @@ from robotmbt import SuiteProcessor -class MyProcessor(SuiteProcessor): +class MyProcessor(SuiteProcessor): def process_test_suite(self, in_suite): self.in_suite = in_suite self._fail_on_step_errors() diff --git a/atest/robotMBT tests/07__processor_options/option_handling/strictsuiterepeater.py b/atest/robotMBT tests/07__processor_options/option_handling/strictsuiterepeater.py index fd38dd8..6344afb 100644 --- a/atest/robotMBT tests/07__processor_options/option_handling/strictsuiterepeater.py +++ b/atest/robotMBT tests/07__processor_options/option_handling/strictsuiterepeater.py @@ -2,6 +2,7 @@ from suiterepeater import SuiteRepeater + @library(auto_keywords=None, listener=True) class StrictSuiteRepeater(SuiteRepeater): """ @@ -9,5 +10,6 @@ class StrictSuiteRepeater(SuiteRepeater): this variant is strict in its argument handling and will fail if mandatory arguments are missing or unknown arguments are provided. """ + def process_test_suite(self, in_suite, *, repeat, bonus_scenario=False): return super().process_test_suite(in_suite, repeat=repeat, bonus_scenario=bonus_scenario) diff --git a/atest/robotMBT tests/07__processor_options/option_handling/suiterepeater.py b/atest/robotMBT tests/07__processor_options/option_handling/suiterepeater.py index 1679f43..dde9c95 100644 --- a/atest/robotMBT tests/07__processor_options/option_handling/suiterepeater.py +++ b/atest/robotMBT tests/07__processor_options/option_handling/suiterepeater.py @@ -3,6 +3,7 @@ from robot.api.deco import library from robotmbt import SuiteProcessor + @library(auto_keywords=None, listener=True) class SuiteRepeater(SuiteProcessor): """ From 70935a28e8cfd96238ff546edc85187d8597a619 Mon Sep 17 00:00:00 2001 From: JFoederer <32476108+JFoederer@users.noreply.github.com> Date: Wed, 10 Jun 2026 08:59:15 +0200 Subject: [PATCH 08/19] add scenario count method for suites --- robotmbt/suitedata.py | 3 +++ utest/test_suitedata.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/robotmbt/suitedata.py b/robotmbt/suitedata.py index 913f0f3..359b9fb 100644 --- a/robotmbt/suitedata.py +++ b/robotmbt/suitedata.py @@ -69,6 +69,9 @@ def steps_with_errors(self): + [e for s in map(Scenario.steps_with_errors, self.scenarios) for e in s] + ([self.teardown] if self.teardown and self.teardown.has_error() else [])) + def scenario_count(self): + return len(self.scenarios) + sum([s.scenario_count() for s in self.suites]) + class Scenario: def __init__(self, name: str, parent: Suite, og_tc): diff --git a/utest/test_suitedata.py b/utest/test_suitedata.py index 691f339..6772696 100644 --- a/utest/test_suitedata.py +++ b/utest/test_suitedata.py @@ -66,6 +66,9 @@ def test_longname_with_parent_includes_all_parent_names(self): self.assertEqual(self.topsuite.suites[-1].scenarios[-1].longname, 'topsuite.suite B.scenario BB') + def test_scenario_count(self): + self.assertEqual(self.topsuite.scenario_count(), 6) + def test_error_in_suite_setup_is_detected(self): step = Step('top setup', parent=self.topsuite) step.gherkin_kw = 'given' From 9691fd7d54d3139de72123b4ccf1120a316c33be Mon Sep 17 00:00:00 2001 From: JFoederer <32476108+JFoederer@users.noreply.github.com> Date: Wed, 10 Jun 2026 15:11:47 +0200 Subject: [PATCH 09/19] centralise target option handling --- .../03__parse_model_info/MyProcessor.py | 3 +- .../option_handling/suiterepeater.py | 2 + robotmbt/suiteprocessors.py | 107 +++++++++++++----- robotmbt/suitereplacer.py | 7 +- robotmbt/tracestate.py | 9 +- utest/test_tracestate.py | 24 ++++ 6 files changed, 117 insertions(+), 35 deletions(-) diff --git a/atest/robotMBT tests/03__parse_model_info/MyProcessor.py b/atest/robotMBT tests/03__parse_model_info/MyProcessor.py index ac2d9a0..c2bd9d7 100644 --- a/atest/robotMBT tests/03__parse_model_info/MyProcessor.py +++ b/atest/robotMBT tests/03__parse_model_info/MyProcessor.py @@ -2,7 +2,8 @@ class MyProcessor(SuiteProcessor): - def process_test_suite(self, in_suite): + def process_test_suite(self, in_suite, **kwargs): + super().process_test_suite(in_suite, **kwargs) self.in_suite = in_suite self._fail_on_step_errors() msg = "Model info not properly parsed" diff --git a/atest/robotMBT tests/07__processor_options/option_handling/suiterepeater.py b/atest/robotMBT tests/07__processor_options/option_handling/suiterepeater.py index dde9c95..ede1ede 100644 --- a/atest/robotMBT tests/07__processor_options/option_handling/suiterepeater.py +++ b/atest/robotMBT tests/07__processor_options/option_handling/suiterepeater.py @@ -13,9 +13,11 @@ class SuiteRepeater(SuiteProcessor): """ def process_test_suite(self, in_suite, repeat=1, **kwargs): + super().process_test_suite(in_suite, **kwargs) n_repeats = int(repeat) if kwargs.get('bonus_scenario', False): n_repeats += 1 + self.scenario_count *= n_repeats out_suite = copy.deepcopy(in_suite) out_suite.scenarios = n_repeats*out_suite.scenarios for i in range(len(out_suite.scenarios)): diff --git a/robotmbt/suiteprocessors.py b/robotmbt/suiteprocessors.py index ab09c08..b742a46 100644 --- a/robotmbt/suiteprocessors.py +++ b/robotmbt/suiteprocessors.py @@ -48,25 +48,65 @@ class SuiteProcessor: - def process_test_suite(self, in_suite: Suite, **kwargs) -> Suite: - raise NotImplementedError() + def process_test_suite(self, in_suite: Suite, + **kwargs) -> Suite: + self._handle_target_options(**kwargs) + self.scenario_count = in_suite.scenario_count() + # Counts the scenarios committed by the runner. I.e. the scenarios that are scheduled for execution + # and cannot be touched anymore + self.commit_count: int = 0 + return Suite('not implemented') + + def next_scenario_request(self) -> int: + """ + Indicates the wish for (at least) one more scenario to trigger trace genaration when needed. + Returns the number of buffered scenarios. If 0 is returned, nothing could be added anymore. + """ + # This basic implementation assumes that the complete target test suite is returned directly + # by process_test_suite() in an overridden method. No further generation is triggered. + return self.scenario_count - self.commit_count - def next_scenario_request(self): - pass + def commit_next_scenario(self): + self.commit_count += 1 + + def are_all_targets_reached(self, committed_only: bool = True) -> bool: + if not committed_only: + return True + if self.coverage_target and self.commit_count < self.scenario_count: + return False + if self.scenario_target and self.commit_count < self.scenario_target: + return False + return True + + def _handle_target_options(self, + coverage_target: str | int | None = 1, + scenario_target: str | int | None = None, + **kwargs): + self.coverage_target = 0 if coverage_target is None else int(coverage_target) + if self.coverage_target not in [0, 1]: + logger.warn(f"Unsuppported coverage target request '{coverage_target}'. Using default coverage target of 1") + self.coverage_target = 1 + self.scenario_target = 0 if scenario_target is None else int(scenario_target) class Echo(SuiteProcessor): - def process_test_suite(self, in_suite: Suite) -> Suite: + def process_test_suite(self, in_suite: Suite, **kwargs) -> Suite: + super().process_test_suite(in_suite, **kwargs) return in_suite class Flatten(SuiteProcessor): - def process_test_suite(self, in_suite: Suite) -> Suite: + def process_test_suite(self, in_suite: Suite, **kwargs) -> Suite: """ Takes a Suite as input and returns a Suite as output. The output Suite does not have any sub-suites, only scenarios. The scenarios do not have a setup. Any setup keywords are inserted at the front of the scenario as regular steps. """ + super().process_test_suite(in_suite, **kwargs) + return self.flatten(in_suite) + + @staticmethod + def flatten(in_suite: Suite) -> Suite: out_suite = copy.deepcopy(in_suite) outer_scenarios = out_suite.scenarios for scenario in outer_scenarios: @@ -78,7 +118,7 @@ def process_test_suite(self, in_suite: Suite) -> Suite: scenario.teardown = None out_suite.scenarios = [] for suite in in_suite.suites: - subsuite = self.process_test_suite(suite) + subsuite = Flatten.flatten(suite) for scenario in subsuite.scenarios: if subsuite.setup: scenario.steps.insert(0, subsuite.setup) @@ -93,30 +133,24 @@ def process_test_suite(self, in_suite: Suite) -> Suite: class ModelBased(SuiteProcessor): def process_test_suite(self, in_suite: Suite, *, seed: str | int | bytes | bytearray = 'new', batch_size: str | int = 100, - coverage_target: str | int | None = 1, - scenario_target: str | int | None = None, - graph: str = '', export_graph_data: str = '') -> Suite: + graph: str = '', export_graph_data: str = '', **kwargs) -> Suite: + # handle options + super().process_test_suite(in_suite, **kwargs) + self.batch_size = int(batch_size) + self._init_randomiser(seed) + self._visualiser = self._init_visualiser(in_suite.name) if graph or export_graph_data else None + self.out_suite = Suite(in_suite.name) self.out_suite.filename = in_suite.filename self.out_suite.parent = in_suite.parent self._fail_on_step_errors(in_suite) - self.flat_suite = Flatten().process_test_suite(in_suite) + self.flat_suite = Flatten.flatten(in_suite) for id, scenario in enumerate(self.flat_suite.scenarios, start=1): scenario.src_id = id self.scenarios: list[Scenario] = self.flat_suite.scenarios[:] logger.debug("Use these numbers to reference scenarios from traces\n\t" + "\n\t".join([f"{s.src_id}: {s.name}" for s in self.scenarios])) - # handle options - self.batch_size = int(batch_size) - self.coverage_target = 0 if coverage_target is None else int(coverage_target) - if self.coverage_target not in [0, 1]: - logger.warn(f"Unsuppported coverage target request '{coverage_target}'. Using default coverage target of 1") - self.coverage_target = 1 - self.scenario_target = 0 if scenario_target is None else int(scenario_target) - self._init_randomiser(seed) - self._visualiser = self._init_visualiser(in_suite.name) if graph or export_graph_data else None - try: # a short trace without the need for repeating scenarios is preferred direct_tracestate = self._search_direct_trace() @@ -139,17 +173,32 @@ def process_test_suite(self, in_suite: Suite, *, seed: str | int | bytes | bytea if len(self.tracestate) == 0: raise Exception("Unable to compose a consistent suite") self._report_tracestate_wrapup(self.tracestate) - self.index = 0 return self.out_suite def next_scenario_request(self): - if len(self.tracestate) <= self.index: + if len(self.tracestate) <= self.commit_count: self._generate_next_batch(self.batch_size) logger.warn(f"Extending run with max. {self.batch_size} scenarios. Now {len(self.tracestate)} long.") - if len(self.tracestate) > self.index: - self.out_suite.scenarios.append(self.tracestate[self.index].scenario) - self.index += 1 - self.tracestate.rewind_limit += 1 + if len(self.tracestate) > self.commit_count: + self.out_suite.scenarios.append(self.tracestate[self.commit_count].scenario) + return len(self.tracestate) - self.commit_count + + def commit_next_scenario(self): + self.commit_count += 1 + self.tracestate.rewind_limit += 1 + + def are_all_targets_reached(self, committed_only: bool = True) -> bool: + if committed_only: + if self.coverage_target and not self.tracestate[self.commit_count-1].coverage_reached: + return False + if self.scenario_target and self.commit_count < self.scenario_target: + return False + else: + if self.coverage_target and not self.tracestate.coverage_reached(): + return False + if self.scenario_target and len(self.tracestate) < self.scenario_target: + return False + return True def draw_graph_from_export_file(self, file_path: str, graph_style: str): self._visualiser = self._init_visualiser() @@ -300,9 +349,7 @@ def _generate_next_batch(self, batchsize): tracestate = self.tracestate old_len = len(tracestate) self._update_visualisation(tracestate) - while len(tracestate) < old_len + batchsize and \ - (self.coverage_target and not tracestate.coverage_reached() - or self.scenario_target and len(tracestate) < self.scenario_target): + while len(tracestate) < old_len + batchsize and not self.are_all_targets_reached(committed_only=False): candidate_id = tracestate.next_candidate(retry=True, randomise=True) if candidate_id is None: # No more candidates remaining for this level if not tracestate.can_rewind(): diff --git a/robotmbt/suitereplacer.py b/robotmbt/suitereplacer.py index 9e2c443..24a3d63 100644 --- a/robotmbt/suitereplacer.py +++ b/robotmbt/suitereplacer.py @@ -91,8 +91,8 @@ def treat_model_based(self, **kwargs): self.suite_gen = [iter(modelbased_suite.suites)] self.test_case_gen = [iter(modelbased_suite.scenarios)] self.__clearTestSuite(self.current_suite) - self.add_next_new(self.current_suite) # add first test case only. Others are added at runtime by listeners. self.mbt_anchor_suite = self.current_suite + self.add_next_new(self.mbt_anchor_suite) @keyword("Set model-based options") def set_model_based_options(self, **kwargs): @@ -182,6 +182,7 @@ def add_next_new(self, target_suite: robot.model.TestSuite): self.processor.next_scenario_request() try: self.add_test(next(self.test_case_gen[-1]), target_suite) + self.processor.commit_next_scenario() except StopIteration: pass @@ -235,8 +236,12 @@ def _end_test(self, test_case: robot.model.TestCase, result): return if not isinstance(self.processor, SuiteProcessor): raise TypeError("processor must be of type SuiteProcessor") + if self.processor.are_all_targets_reached(): + return + self.processor.next_scenario_request() try: self.add_test(next(self.test_case_gen[-1]), self.current_suite) + self.processor.commit_next_scenario() except StopIteration: pass diff --git a/robotmbt/tracestate.py b/robotmbt/tracestate.py index f907764..970a533 100644 --- a/robotmbt/tracestate.py +++ b/robotmbt/tracestate.py @@ -38,11 +38,12 @@ class TraceSnapShot: def __init__(self, id: str, inserted_scenario: Scenario, model_state: ModelSpace, - remainder: Scenario | None = None, drought: int = 0): + remainder: Scenario | None = None, coverage: int = 0, drought: int = 0): self.id: str = id self.scenario: Scenario = inserted_scenario self.remainder: Scenario | None = remainder self._model: ModelSpace = model_state.copy() + self.coverage_reached: int = coverage self.coverage_drought: int = drought @property @@ -200,7 +201,8 @@ def confirm_full_scenario(self, index: int, scenario: Scenario, model: ModelSpac id = str(index) self._tried[-1].append(index) self._tried.append([]) - self._snapshots.append(TraceSnapShot(id, scenario, model, drought=c_drought)) + self._snapshots.append(TraceSnapShot(id, scenario, model, + coverage=min(self.c_pool.values()), drought=c_drought)) def push_partial_scenario(self, index: int, scenario: Scenario, model: ModelSpace, remainder=None): if self.is_refinement_active(index): @@ -210,7 +212,8 @@ def push_partial_scenario(self, index: int, scenario: Scenario, model: ModelSpac self._tried[-1].append(index) self._open_refinements.append(index) self._tried.append([]) - self._snapshots.append(TraceSnapShot(id, scenario, model, remainder, self.coverage_drought)) + self._snapshots.append(TraceSnapShot(id, scenario, model, remainder, + coverage=min(self.c_pool.values()), drought=self.coverage_drought)) def can_rewind(self) -> bool: rewind_margin = len(self._snapshots[self.rewind_limit:]) diff --git a/utest/test_tracestate.py b/utest/test_tracestate.py index f78b883..db2f241 100644 --- a/utest/test_tracestate.py +++ b/utest/test_tracestate.py @@ -305,6 +305,30 @@ def test_can_index_tracestate_snapshots(self): self.assertEqual(ts[-1].scenario, 'three') self.assertEqual([s.id for s in ts[1:]], ['2', '3']) + def test_tracestate_snapshots_track_coverage(self): + """ + Coverage counter shows the number of times that full coverage is achieved. I.e., the + counter stays 0 until the last sceanrio is reached for the first time. Then it stays + at 1 until all scenarios have been executed at least a second time. + """ + ts = TraceState([1, 2, 3]) + ts.confirm_full_scenario(1, ScenarioStub('one A'), ModelStub()) + ts.confirm_full_scenario(2, ScenarioStub('two A'), ModelStub()) + ts.confirm_full_scenario(3, ScenarioStub('three A'), ModelStub()) + self.assertEqual(ts[-1].coverage_reached, 1) + self.assertEqual(ts[-2].coverage_reached, 0) + self.assertEqual(ts[0].coverage_reached, 0) + ts.confirm_full_scenario(1, ScenarioStub('one B'), ModelStub()) + ts.confirm_full_scenario(2, ScenarioStub('two B'), ModelStub()) + ts.confirm_full_scenario(3, ScenarioStub('three B'), ModelStub()) + self.assertEqual(ts[1].coverage_reached, 0) + self.assertEqual(ts[2].coverage_reached, 1) + self.assertEqual(ts[-1].coverage_reached, 2) + ts.confirm_full_scenario(3, ScenarioStub('three C'), ModelStub()) + self.assertEqual(ts[-1].coverage_reached, 2) + self.assertEqual(ts[-2].coverage_reached, 2) + self.assertEqual(ts[-3].coverage_reached, 1) + def test_adding_coverage_prevents_drought(self): ts = TraceState(range(3)) ts.confirm_full_scenario(ts.next_candidate(), ScenarioStub('one'), ModelStub()) From 5cb4b50fd17411685b2cb10534c9b910a7a94306 Mon Sep 17 00:00:00 2001 From: JFoederer <32476108+JFoederer@users.noreply.github.com> Date: Fri, 12 Jun 2026 13:30:56 +0200 Subject: [PATCH 10/19] add stop condition tests --- ...__stop_at_single_coverage_by_default.robot | 26 +++++++++++++++++++ .../02__stop_beyond_single_coverage.robot | 26 +++++++++++++++++++ .../03__stop_before_single_coverage.robot | 26 +++++++++++++++++++ robotmbt/suiteprocessors.py | 2 +- 4 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 atest/robotMBT tests/07__processor_options/stop_conditions/01__stop_at_single_coverage_by_default.robot create mode 100644 atest/robotMBT tests/07__processor_options/stop_conditions/02__stop_beyond_single_coverage.robot create mode 100644 atest/robotMBT tests/07__processor_options/stop_conditions/03__stop_before_single_coverage.robot diff --git a/atest/robotMBT tests/07__processor_options/stop_conditions/01__stop_at_single_coverage_by_default.robot b/atest/robotMBT tests/07__processor_options/stop_conditions/01__stop_at_single_coverage_by_default.robot new file mode 100644 index 0000000..cbdd4b0 --- /dev/null +++ b/atest/robotMBT tests/07__processor_options/stop_conditions/01__stop_at_single_coverage_by_default.robot @@ -0,0 +1,26 @@ +*** Settings *** +Documentation At single coverage (the default), the final suite should repeat the middle +... exactly once, then insert the last scenario and stop. +Suite Setup Treat this test suite Model-based +Suite Teardown Should be equal ${scenario_count} ${4} +Test Teardown Set suite variable ${scenario_count} ${scenario_count+1} +Resource ../../../resources/birthday_cards_flat.resource +Library robotmbt + +*** variables *** +${scenario_count} ${0} + +*** Test Cases *** +Buying a card + When someone buys a birthday card + then there is a blank birthday card available + +Someone writes their name on the card + Given there is a birthday card + when Someone writes their name on the birthday card + then the birthday card has 'Someone' written on it + +At least 3 people can write their name on the card + Given the birthday card has 2 names written on it + when someone writes their name on the birthday card + then the birthday card has 3 names written on it diff --git a/atest/robotMBT tests/07__processor_options/stop_conditions/02__stop_beyond_single_coverage.robot b/atest/robotMBT tests/07__processor_options/stop_conditions/02__stop_beyond_single_coverage.robot new file mode 100644 index 0000000..9259008 --- /dev/null +++ b/atest/robotMBT tests/07__processor_options/stop_conditions/02__stop_beyond_single_coverage.robot @@ -0,0 +1,26 @@ +*** Settings *** +Documentation Create a test trace that extends beyond single coverage by using a scenario +... target that is one higher than needed to reach single coverage. +Suite Setup Treat this test suite Model-based scenario_target=5 +Suite Teardown Should be equal ${scenario_count} ${5} +Test Teardown Set suite variable ${scenario_count} ${scenario_count+1} +Resource ../../../resources/birthday_cards_flat.resource +Library robotmbt + +*** variables *** +${scenario_count} ${0} + +*** Test Cases *** +Buying a card + When someone buys a birthday card + then there is a blank birthday card available + +Someone writes their name on the card + Given there is a birthday card + when Someone writes their name on the birthday card + then the birthday card has 'Someone' written on it + +At least 3 people can write their name on the card + Given the birthday card has 2 names written on it + when someone writes their name on the birthday card + then the birthday card has 3 names written on it diff --git a/atest/robotMBT tests/07__processor_options/stop_conditions/03__stop_before_single_coverage.robot b/atest/robotMBT tests/07__processor_options/stop_conditions/03__stop_before_single_coverage.robot new file mode 100644 index 0000000..4c30032 --- /dev/null +++ b/atest/robotMBT tests/07__processor_options/stop_conditions/03__stop_before_single_coverage.robot @@ -0,0 +1,26 @@ +*** Settings *** +Documentation Create a test trace that stops before single coverage is reached by using a +... scenario target that is lower than what is needed to reach single coverage. +Suite Setup Treat this test suite Model-based coverage_target=0 scenario_target=2 +Suite Teardown Should be equal ${scenario_count} ${2} +Test Teardown Set suite variable ${scenario_count} ${scenario_count+1} +Resource ../../../resources/birthday_cards_flat.resource +Library robotmbt + +*** variables *** +${scenario_count} ${0} + +*** Test Cases *** +Buying a card + When someone buys a birthday card + then there is a blank birthday card available + +Someone writes their name on the card + Given there is a birthday card + when Someone writes their name on the birthday card + then the birthday card has 'Someone' written on it + +At least 3 people can write their name on the card + Given the birthday card has 2 names written on it + when someone writes their name on the birthday card + then the birthday card has 3 names written on it diff --git a/robotmbt/suiteprocessors.py b/robotmbt/suiteprocessors.py index b742a46..f9523d4 100644 --- a/robotmbt/suiteprocessors.py +++ b/robotmbt/suiteprocessors.py @@ -84,7 +84,7 @@ def _handle_target_options(self, **kwargs): self.coverage_target = 0 if coverage_target is None else int(coverage_target) if self.coverage_target not in [0, 1]: - logger.warn(f"Unsuppported coverage target request '{coverage_target}'. Using default coverage target of 1") + logger.warn(f"Unsupported coverage target request '{coverage_target}'. Using default coverage target of 1") self.coverage_target = 1 self.scenario_target = 0 if scenario_target is None else int(scenario_target) From 4b0b51bc13f2c5f4a5612b4811bbbf40459532b1 Mon Sep 17 00:00:00 2001 From: JFoederer <32476108+JFoederer@users.noreply.github.com> Date: Sat, 13 Jun 2026 11:55:33 +0200 Subject: [PATCH 11/19] let direct trace respect targets as well --- robotmbt/suiteprocessors.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/robotmbt/suiteprocessors.py b/robotmbt/suiteprocessors.py index f9523d4..c997420 100644 --- a/robotmbt/suiteprocessors.py +++ b/robotmbt/suiteprocessors.py @@ -154,12 +154,11 @@ def process_test_suite(self, in_suite: Suite, *, seed: str | int | bytes | bytea try: # a short trace without the need for repeating scenarios is preferred direct_tracestate = self._search_direct_trace() - if direct_tracestate.coverage_reached(): + if self.are_all_targets_reached(direct_tracestate, committed_only=False): # The visualiser assumes that the last trace is the final selected trace, which is not always # the case. Re-adding the selected trace to prevent the wrong path from being highlighted. self.tracestate = direct_tracestate self._update_visualisation(self.tracestate) - self._update_visualisation(self.tracestate) else: self.tracestate = TraceState([s.src_id for s in self.scenarios]) self.tracestate.unreached = direct_tracestate.unreached @@ -172,6 +171,7 @@ def process_test_suite(self, in_suite: Suite, *, seed: str | int | bytes | bytea self._export_graph_data(export_graph_data) if len(self.tracestate) == 0: raise Exception("Unable to compose a consistent suite") + self.out_suite.scenarios = self.tracestate.get_trace() self._report_tracestate_wrapup(self.tracestate) return self.out_suite @@ -187,16 +187,18 @@ def commit_next_scenario(self): self.commit_count += 1 self.tracestate.rewind_limit += 1 - def are_all_targets_reached(self, committed_only: bool = True) -> bool: + def are_all_targets_reached(self, tracestate: TraceState | None = None, committed_only: bool = True) -> bool: + if tracestate is None: + tracestate = self.tracestate if committed_only: - if self.coverage_target and not self.tracestate[self.commit_count-1].coverage_reached: + if self.coverage_target and not tracestate[self.commit_count-1].coverage_reached: return False if self.scenario_target and self.commit_count < self.scenario_target: return False else: - if self.coverage_target and not self.tracestate.coverage_reached(): + if self.coverage_target and not tracestate.coverage_reached(): return False - if self.scenario_target and len(self.tracestate) < self.scenario_target: + if self.scenario_target and len(tracestate) < self.scenario_target: return False return True @@ -238,23 +240,23 @@ def _search_direct_trace(self) -> TraceState: if self._is_duplicate_prio_order(tracestates, prio_order): continue tracestates.append(self._one_shot_trace(prio_order)) - if tracestates[-1].coverage_reached() and not self._visualiser: + if self.are_all_targets_reached(tracestates[-1], committed_only=False) and not self._visualiser: return tracestates[-1] suggestion = self._create_suggestion_by_experience(tracestates) if self._is_duplicate_prio_order(tracestates, suggestion): continue tracestates.append(self._one_shot_trace(suggestion)) - if tracestates[-1].coverage_reached() and not self._visualiser: + if self.are_all_targets_reached(tracestates[-1], committed_only=False) and not self._visualiser: return tracestates[-1] index_longest = self._longest_trace(tracestates) - if tracestates[index_longest].coverage_reached(): + if self.are_all_targets_reached(tracestates[index_longest], committed_only=False): return tracestates[index_longest] logger.debug("Trying to extend most promising traces") prio_order = self._create_suggestion_by_experience(tracestates, index_longest) if not self._is_duplicate_prio_order(tracestates, prio_order): tracestates.append(self._one_shot_trace(prio_order)) - if tracestates[-1].coverage_reached(): + if self.are_all_targets_reached(tracestates[-1], committed_only=False): return tracestates[-1] last_new = self._last_new_coverage(tracestates) while True: # while still discovering new coverage @@ -262,7 +264,7 @@ def _search_direct_trace(self) -> TraceState: if self._is_duplicate_prio_order(tracestates, prio_order): break tracestates.append(self._one_shot_trace(prio_order)) - if tracestates[-1].coverage_reached(): + if self.are_all_targets_reached(tracestates[-1], committed_only=False): return tracestates[-1] last_new = self._last_new_coverage(tracestates) if last_new != len(tracestates)-1: From f599051104567a59f0977fe35899a42508e5caa0 Mon Sep 17 00:00:00 2001 From: JFoederer <32476108+JFoederer@users.noreply.github.com> Date: Sun, 14 Jun 2026 19:14:26 +0200 Subject: [PATCH 12/19] testing batch sizes --- ...st_complete_to_reach_coverage_target.robot | 41 ++++++ ...tay_incomplete_once_coverage_reached.robot | 50 +++++++ robotmbt/suiteprocessors.py | 9 +- utest/test_suiteprocessors.py | 126 ++++++++++++++++++ 4 files changed, 221 insertions(+), 5 deletions(-) create mode 100644 atest/robotMBT tests/07__processor_options/stop_conditions/04__refinement_must_complete_to_reach_coverage_target.robot create mode 100644 atest/robotMBT tests/07__processor_options/stop_conditions/05__refinement_can _stay_incomplete_once_coverage_reached.robot diff --git a/atest/robotMBT tests/07__processor_options/stop_conditions/04__refinement_must_complete_to_reach_coverage_target.robot b/atest/robotMBT tests/07__processor_options/stop_conditions/04__refinement_must_complete_to_reach_coverage_target.robot new file mode 100644 index 0000000..5384b60 --- /dev/null +++ b/atest/robotMBT tests/07__processor_options/stop_conditions/04__refinement_must_complete_to_reach_coverage_target.robot @@ -0,0 +1,41 @@ +*** Settings *** +Documentation Due to refinement, the high-level scenario is split up to insert the low-level +... scenario. Coverage is not completed until the final part of the split-up +... scenario is executed. +Suite Setup Treat this test suite Model-based coverage_target=1 +Suite Teardown Final checks +Test Teardown Set suite variable ${scenario_count} ${scenario_count+1} +Resource ../../../resources/birthday_cards_composed.resource +Library robotmbt + + +*** variables *** +${scenario_count} ${0} +${high_level_started} ${0} +${high_level_completed} ${0} + + +*** Test Cases *** +Buying a card + When someone buys a birthday card + then there is a blank birthday card available + +high-level scenario + Set suite variable ${high_level_started} ${high_level_started+1} + Given there is a birthday card + when Someone writes their name on the birthday card + then the birthday card has 'Someone' written on it + Set suite variable ${high_level_completed} ${high_level_completed+1} + +low-level scenario + Given there is a birthday card + when Someone writes their name in pen on the birthday card + then the birthday card has 'Someone' written on it + and there is text added in ink on the birthday card + + +*** Keywords *** +Final checks + Should be equal ${scenario_count} ${3} + Should be equal ${high_level_started} ${1} + Should be equal ${high_level_completed} ${1} diff --git a/atest/robotMBT tests/07__processor_options/stop_conditions/05__refinement_can _stay_incomplete_once_coverage_reached.robot b/atest/robotMBT tests/07__processor_options/stop_conditions/05__refinement_can _stay_incomplete_once_coverage_reached.robot new file mode 100644 index 0000000..3b478b5 --- /dev/null +++ b/atest/robotMBT tests/07__processor_options/stop_conditions/05__refinement_can _stay_incomplete_once_coverage_reached.robot @@ -0,0 +1,50 @@ +*** Settings *** +Documentation Due to refinement, the high-level scenario is split up to insert a low-level +... scenario. Because there are two low-level scenarios, the high-level scenario +... must be repeated. Coverage for the high-level scenario is reached as a soon +... as the first one completes. Therefore, full coverage is reached as soon as the +... second lower-level scenario completes, causing the trace to end, even though +... that high-level scenario did not complete yet. +Suite Setup Treat this test suite Model-based coverage_target=1 +Suite Teardown Final checks +Test Teardown Set suite variable ${scenario_count} ${scenario_count+1} +Resource ../../../resources/birthday_cards_composed.resource +Library robotmbt + + +*** variables *** +${scenario_count} ${0} +${high_level_started} ${0} +${high_level_completed} ${0} + + +*** Test Cases *** +Buying a card + When someone buys a birthday card + then there is a blank birthday card available + +high-level scenario + Set suite variable ${high_level_started} ${high_level_started+1} + Given there is a birthday card + when Someone writes their name on the birthday card + then the birthday card has 'Someone' written on it + Set suite variable ${high_level_completed} ${high_level_completed+1} + +low-level scenario A + Given there is a birthday card + when Someone writes their name in pen on the birthday card + then the birthday card has 'Someone' written on it + and there is text added in ink on the birthday card + +low-level scenario B + Given there is a birthday card + when Someone writes their name in pen on the birthday card + then the birthday card has 'Someone' written on it + and there is text added in ink on the birthday card + + +*** Keywords *** +Final checks + Should be equal ${scenario_count} ${4} + Should be equal ${high_level_started} ${2} + Should be equal ${high_level_completed} ${1} diff --git a/robotmbt/suiteprocessors.py b/robotmbt/suiteprocessors.py index c997420..fb1f7ef 100644 --- a/robotmbt/suiteprocessors.py +++ b/robotmbt/suiteprocessors.py @@ -171,21 +171,20 @@ def process_test_suite(self, in_suite: Suite, *, seed: str | int | bytes | bytea self._export_graph_data(export_graph_data) if len(self.tracestate) == 0: raise Exception("Unable to compose a consistent suite") - self.out_suite.scenarios = self.tracestate.get_trace() self._report_tracestate_wrapup(self.tracestate) return self.out_suite def next_scenario_request(self): - if len(self.tracestate) <= self.commit_count: + if len(self.tracestate) <= self.out_suite.scenario_count(): self._generate_next_batch(self.batch_size) logger.warn(f"Extending run with max. {self.batch_size} scenarios. Now {len(self.tracestate)} long.") - if len(self.tracestate) > self.commit_count: - self.out_suite.scenarios.append(self.tracestate[self.commit_count].scenario) - return len(self.tracestate) - self.commit_count + if len(self.tracestate) > self.out_suite.scenario_count(): + self.out_suite.scenarios.append(self.tracestate[self.out_suite.scenario_count()].scenario) def commit_next_scenario(self): self.commit_count += 1 self.tracestate.rewind_limit += 1 + return len(self.tracestate) - self.commit_count def are_all_targets_reached(self, tracestate: TraceState | None = None, committed_only: bool = True) -> bool: if tracestate is None: diff --git a/utest/test_suiteprocessors.py b/utest/test_suiteprocessors.py index e616049..b818810 100644 --- a/utest/test_suiteprocessors.py +++ b/utest/test_suiteprocessors.py @@ -34,6 +34,7 @@ from unittest.mock import patch, call from robotmbt.suiteprocessors import ModelBased +from robotmbt.suitedata import Suite, Scenario, Step @patch('robotmbt.suiteprocessors.random.seed') @@ -104,5 +105,130 @@ def _is_generated_seed(self, arg): self.assertTrue(3 <= len(word) <= 6) +class TestBatchSize(unittest.TestCase): + def setUp(self): + self.suite = Suite('testsuite') + init_scenario = Scenario('init scenario', self.suite, RobotTestCaseStub()) + init_step = Step('init keyword', parent=init_scenario) + init_step.model_info = dict(IN=["new prop"], OUT=["prop.flag = True"]) + init_scenario.steps = [init_step] + body_scenario = Scenario('body scenario', self.suite, RobotTestCaseStub()) + self.scenario_name_without_rep_count = len('body scenario') + step = Step('action keyword', parent=body_scenario) + step.model_info = dict(IN=["prop.flag = not prop.flag"], OUT=[]) # force a change so retries are not rejected + body_scenario.steps = [step] + self.suite.scenarios = [init_scenario, body_scenario] + self.processor = ModelBased() + + def test_batch_size_1(self): + out_suite = self.processor.process_test_suite(self.suite, scenario_target=3, batch_size=1) + self.processor.next_scenario_request() + self.assertEqual(out_suite.scenario_count(), 1) + buffered = self.processor.commit_next_scenario() + self.assertEqual(buffered, 0) + self.processor.next_scenario_request() + buffered = self.processor.commit_next_scenario() + self.assertEqual(buffered, 0) + self.processor.next_scenario_request() + buffered = self.processor.commit_next_scenario() + self.assertEqual(buffered, 0) + self.assertListEqual([s.name[:self.scenario_name_without_rep_count] for s in out_suite.scenarios], + ['init scenario'] + ['body scenario']*(out_suite.scenario_count()-1)) + + def test_batch_size_2_last_batch_not_full(self): + out_suite = self.processor.process_test_suite(self.suite, scenario_target=3, batch_size=2) + self.processor.next_scenario_request() + self.assertEqual(out_suite.scenario_count(), 1) + buffered = self.processor.commit_next_scenario() + self.assertEqual(buffered, 1) + self.processor.next_scenario_request() + buffered = self.processor.commit_next_scenario() + self.assertEqual(buffered, 0) + self.processor.next_scenario_request() + buffered = self.processor.commit_next_scenario() + self.assertEqual(buffered, 0) + self.assertEqual(out_suite.scenario_count(), 3) + + def test_batch_size_2_last_batch_full(self): + out_suite = self.processor.process_test_suite(self.suite, scenario_target=4, batch_size=2) + self.processor.next_scenario_request() + self.assertEqual(out_suite.scenario_count(), 1) + buffered = self.processor.commit_next_scenario() + self.assertEqual(buffered, 1) + self.processor.next_scenario_request() + buffered = self.processor.commit_next_scenario() + self.assertEqual(buffered, 0) + self.processor.next_scenario_request() + buffered = self.processor.commit_next_scenario() + self.assertEqual(buffered, 1) + self.processor.next_scenario_request() + buffered = self.processor.commit_next_scenario() + self.assertEqual(buffered, 0) + self.assertEqual(out_suite.scenario_count(), 4) + + def test_batch_size_10(self): + out_suite = self.processor.process_test_suite(self.suite, scenario_target=15, batch_size=10) + self.processor.next_scenario_request() + self.assertEqual(out_suite.scenario_count(), 1) + buffered = self.processor.commit_next_scenario() + self.assertEqual(buffered, 9) + for _ in range(9): + self.processor.next_scenario_request() + buffered = self.processor.commit_next_scenario() + self.assertEqual(out_suite.scenario_count(), 10) + self.assertEqual(buffered, 0) + self.processor.next_scenario_request() + buffered = self.processor.commit_next_scenario() + self.assertEqual(buffered, 4) + + def test_batch_size_3_is_trace_length(self): + out_suite = self.processor.process_test_suite(self.suite, scenario_target=3, batch_size=3) + self.processor.next_scenario_request() + buffered = self.processor.commit_next_scenario() + self.assertEqual(buffered, 2) + self.processor.next_scenario_request() + self.processor.next_scenario_request() + self.assertEqual(out_suite.scenario_count(), 3) + self.assertListEqual([s.name[:self.scenario_name_without_rep_count] for s in out_suite.scenarios], + ['init scenario'] + ['body scenario']*(out_suite.scenario_count()-1)) + + def test_requesting_beyond_targets_has_no_effect(self): + out_suite = self.processor.process_test_suite(self.suite, scenario_target=3, batch_size=3) + self.processor.next_scenario_request() + self.processor.commit_next_scenario() + self.processor.next_scenario_request() + self.processor.commit_next_scenario() + self.assertFalse(self.processor.are_all_targets_reached()) + self.processor.next_scenario_request() + self.processor.commit_next_scenario() + self.assertEqual(out_suite.scenario_count(), 3) + self.assertTrue(self.processor.are_all_targets_reached()) + self.processor.next_scenario_request() + self.assertTrue(self.processor.are_all_targets_reached()) + self.assertEqual(out_suite.scenario_count(), 3) + + def test_multi_batch(self): + """Check some variations in batch size versus target size""" + for target, batch in [(1, 1), # Smallest target and batch + (23, 3), # Multiple batches needed te completer + (10, 24), # Batch size exceeds target size + (15, 14), # Batch size just not enough + (16, 16) # Batch size equals target size + ]: + out_suite = self.processor.process_test_suite(self.suite, coverage_target=0, + scenario_target=target, batch_size=batch) + while not self.processor.are_all_targets_reached(): + self.processor.next_scenario_request() + self.processor.commit_next_scenario() + self.assertEqual(out_suite.scenario_count(), target) + self.assertListEqual([s.name[:self.scenario_name_without_rep_count] for s in out_suite.scenarios], + ['init scenario'] + ['body scenario']*(out_suite.scenario_count()-1)) + + +class RobotTestCaseStub: + def copy(self, **kwargs): + pass + + if __name__ == '__main__': unittest.main() From 82a7b29d41d656bd3911de8f05f133eee9fa771c Mon Sep 17 00:00:00 2001 From: JFoederer <32476108+JFoederer@users.noreply.github.com> Date: Wed, 17 Jun 2026 10:34:58 +0200 Subject: [PATCH 13/19] tag tests that trigger trace extension --- ...ay_incomplete_once_coverage_reached.robot} | 0 robotmbt/suiteprocessors.py | 63 +++++++++++++------ robotmbt/suitereplacer.py | 27 +++++--- utest/test_suiteprocessors.py | 56 +++++++---------- 4 files changed, 88 insertions(+), 58 deletions(-) rename atest/robotMBT tests/07__processor_options/stop_conditions/{05__refinement_can _stay_incomplete_once_coverage_reached.robot => 05__refinement_can_stay_incomplete_once_coverage_reached.robot} (100%) diff --git a/atest/robotMBT tests/07__processor_options/stop_conditions/05__refinement_can _stay_incomplete_once_coverage_reached.robot b/atest/robotMBT tests/07__processor_options/stop_conditions/05__refinement_can_stay_incomplete_once_coverage_reached.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/stop_conditions/05__refinement_can _stay_incomplete_once_coverage_reached.robot rename to atest/robotMBT tests/07__processor_options/stop_conditions/05__refinement_can_stay_incomplete_once_coverage_reached.robot diff --git a/robotmbt/suiteprocessors.py b/robotmbt/suiteprocessors.py index fb1f7ef..fab8817 100644 --- a/robotmbt/suiteprocessors.py +++ b/robotmbt/suiteprocessors.py @@ -57,17 +57,35 @@ def process_test_suite(self, in_suite: Suite, self.commit_count: int = 0 return Suite('not implemented') - def next_scenario_request(self) -> int: - """ - Indicates the wish for (at least) one more scenario to trigger trace genaration when needed. - Returns the number of buffered scenarios. If 0 is returned, nothing could be added anymore. - """ + def next_scenario_request(self): + """Indicates the wish for (at least) one more scenario and triggers trace genaration when needed.""" # This basic implementation assumes that the complete target test suite is returned directly # by process_test_suite() in an overridden method. No further generation is triggered. - return self.scenario_count - self.commit_count + if self.scenario_count >= self.commit_count + 1: + self.commit_count += 1 - def commit_next_scenario(self): - self.commit_count += 1 + @property + def scenarios_committed(self) -> int: + """ + Each time next_scenario_request() is called, you commit to one more scenario, if there is at + least one more scenario available. This can be an already pending scenario, or new trace + generation can be triggered to generate a next scenario (or batch). Committed scenarios are + assumed to have been executed when determining the achieved targets. + """ + return self.commit_count + + @property + def scenarios_pending(self) -> int: + """ + The number of scenarios that are waiting in the buffer for when more scenarios are requested. + + For practical reasons, trace generation can run ahead of the actual test execution, creating + a buffer of pending scenarios ahead of time. + + Note that pending scenarios are still subject to change. Only committed scenarios are frozen + in place. + """ + return self.scenario_count - self.commit_count def are_all_targets_reached(self, committed_only: bool = True) -> bool: if not committed_only: @@ -171,20 +189,24 @@ def process_test_suite(self, in_suite: Suite, *, seed: str | int | bytes | bytea self._export_graph_data(export_graph_data) if len(self.tracestate) == 0: raise Exception("Unable to compose a consistent suite") - self._report_tracestate_wrapup(self.tracestate) + self._report_tracestate_wrapup() return self.out_suite def next_scenario_request(self): if len(self.tracestate) <= self.out_suite.scenario_count(): self._generate_next_batch(self.batch_size) - logger.warn(f"Extending run with max. {self.batch_size} scenarios. Now {len(self.tracestate)} long.") if len(self.tracestate) > self.out_suite.scenario_count(): self.out_suite.scenarios.append(self.tracestate[self.out_suite.scenario_count()].scenario) + self.commit_count += 1 + self.tracestate.rewind_limit += 1 - def commit_next_scenario(self): - self.commit_count += 1 - self.tracestate.rewind_limit += 1 - return len(self.tracestate) - self.commit_count + @property + def scenarios_committed(self) -> int: + return self.out_suite.scenario_count() + + @property + def scenarios_pending(self) -> int: + return len(self.tracestate) - self.out_suite.scenario_count() def are_all_targets_reached(self, tracestate: TraceState | None = None, committed_only: bool = True) -> bool: if tracestate is None: @@ -352,7 +374,8 @@ def _generate_next_batch(self, batchsize): self._update_visualisation(tracestate) while len(tracestate) < old_len + batchsize and not self.are_all_targets_reached(committed_only=False): candidate_id = tracestate.next_candidate(retry=True, randomise=True) - if candidate_id is None: # No more candidates remaining for this level + if candidate_id is None: + logger.debug("No more candidates remaining at this position.") if not tracestate.can_rewind(): break tail = modeller.rewind(tracestate) @@ -426,10 +449,12 @@ def _report_tracestate_to_user(tracestate: TraceState): logger.debug(f"Trace: [{', '.join(tracestate.id_trace)}] Pending: [{pending}]" f"{' Rejected: ' + str(tracestate.tried) if tracestate.tried else ''}") - @staticmethod - def _report_tracestate_wrapup(tracestate: TraceState): - logger.info("Trace composed:") - for progression in tracestate: + def _report_tracestate_wrapup(self): + if self.are_all_targets_reached(committed_only=False): + logger.info("Trace composed:") + else: + logger.info("First part of trace composed: (Check scenarios tagged with `mbt trace extension` for continued generation)") + for progression in self.tracestate: logger.info(progression.scenario.name) logger.debug(f"model\n{progression.model.get_status_text()}\n") diff --git a/robotmbt/suitereplacer.py b/robotmbt/suitereplacer.py index 24a3d63..942c34d 100644 --- a/robotmbt/suitereplacer.py +++ b/robotmbt/suitereplacer.py @@ -30,10 +30,11 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from collections.abc import Callable, Iterator +from collections.abc import Iterator from typing import Any import robot.model +import robot.result import robot.running.model as rmodel from robot.api import logger from robot.api.deco import library, keyword @@ -92,6 +93,7 @@ def treat_model_based(self, **kwargs): self.test_case_gen = [iter(modelbased_suite.scenarios)] self.__clearTestSuite(self.current_suite) self.mbt_anchor_suite = self.current_suite + self.processor.next_scenario_request() self.add_next_new(self.mbt_anchor_suite) @keyword("Set model-based options") @@ -179,10 +181,8 @@ def add_next_new(self, target_suite: robot.model.TestSuite): new_target = self.add_suite(new_suite, target_suite) self.add_next_new(new_target) except StopIteration: - self.processor.next_scenario_request() try: self.add_test(next(self.test_case_gen[-1]), target_suite) - self.processor.commit_next_scenario() except StopIteration: pass @@ -218,10 +218,10 @@ def add_test(tc: Scenario, target_suite: robot.model.TestSuite): new_tc.body.create_keyword(name=step.keyword, assign=step.assign, args=step.posnom_args_str) target_suite.tests.append(new_tc) - def _start_suite(self, suite: robot.model.TestSuite, result): + def _start_suite(self, suite: rmodel.TestSuite, result): self.current_suite = suite - def _end_suite(self, suite: robot.model.TestSuite, result): + def _end_suite(self, suite: rmodel.TestSuite, result): if suite == self.mbt_anchor_suite: self.mbt_anchor_suite = None if not self.mbt_anchor_suite: @@ -231,17 +231,30 @@ def _end_suite(self, suite: robot.model.TestSuite, result): self.current_suite = self.current_suite.parent self.add_next_new(self.current_suite) - def _end_test(self, test_case: robot.model.TestCase, result): + def _end_test(self, test_case: rmodel.TestCase, result: robot.result.model.TestCase): if not self.mbt_anchor_suite: return if not isinstance(self.processor, SuiteProcessor): raise TypeError("processor must be of type SuiteProcessor") if self.processor.are_all_targets_reached(): + logger.info(f"{self.processor.scenarios_committed} Scenarios completed for model. All targets achieved.") return + committed_old = self.processor.scenarios_committed + pending_old = self.processor.scenarios_pending + if not pending_old: + logger.info(f"{committed_old} scenarios completed. Looking to extend trace.") self.processor.next_scenario_request() + committed = self.processor.scenarios_committed + pending = self.processor.scenarios_pending + new_total = committed + pending + old_total = committed_old + pending_old + if new_total > old_total: + result.tags.add('mbt trace extension') + logger.info(f"MBT trace generation added {new_total-old_total} new scenarios.") + if not pending_old and new_total == old_total: + logger.info(f"Trace could not be extended.") try: self.add_test(next(self.test_case_gen[-1]), self.current_suite) - self.processor.commit_next_scenario() except StopIteration: pass diff --git a/utest/test_suiteprocessors.py b/utest/test_suiteprocessors.py index b818810..77be7c9 100644 --- a/utest/test_suiteprocessors.py +++ b/utest/test_suiteprocessors.py @@ -122,70 +122,66 @@ def setUp(self): def test_batch_size_1(self): out_suite = self.processor.process_test_suite(self.suite, scenario_target=3, batch_size=1) + self.assertEqual(self.processor.scenarios_pending, 1) self.processor.next_scenario_request() self.assertEqual(out_suite.scenario_count(), 1) - buffered = self.processor.commit_next_scenario() - self.assertEqual(buffered, 0) + self.assertEqual(self.processor.scenarios_committed, 1) + self.assertEqual(self.processor.scenarios_pending, 0) self.processor.next_scenario_request() - buffered = self.processor.commit_next_scenario() - self.assertEqual(buffered, 0) + self.assertEqual(out_suite.scenario_count(), 2) + self.assertEqual(self.processor.scenarios_committed, 2) + self.assertEqual(self.processor.scenarios_pending, 0) self.processor.next_scenario_request() - buffered = self.processor.commit_next_scenario() - self.assertEqual(buffered, 0) + self.assertEqual(out_suite.scenario_count(), 3) + self.assertEqual(self.processor.scenarios_committed, 3) + self.assertEqual(self.processor.scenarios_pending, 0) self.assertListEqual([s.name[:self.scenario_name_without_rep_count] for s in out_suite.scenarios], ['init scenario'] + ['body scenario']*(out_suite.scenario_count()-1)) def test_batch_size_2_last_batch_not_full(self): out_suite = self.processor.process_test_suite(self.suite, scenario_target=3, batch_size=2) + self.assertEqual(self.processor.scenarios_pending, 2) self.processor.next_scenario_request() self.assertEqual(out_suite.scenario_count(), 1) - buffered = self.processor.commit_next_scenario() - self.assertEqual(buffered, 1) + self.assertEqual(self.processor.scenarios_committed, 1) + self.assertEqual(self.processor.scenarios_pending, 1) self.processor.next_scenario_request() - buffered = self.processor.commit_next_scenario() - self.assertEqual(buffered, 0) + self.assertEqual(self.processor.scenarios_pending, 0) self.processor.next_scenario_request() - buffered = self.processor.commit_next_scenario() - self.assertEqual(buffered, 0) + self.assertEqual(self.processor.scenarios_pending, 0) self.assertEqual(out_suite.scenario_count(), 3) def test_batch_size_2_last_batch_full(self): out_suite = self.processor.process_test_suite(self.suite, scenario_target=4, batch_size=2) self.processor.next_scenario_request() self.assertEqual(out_suite.scenario_count(), 1) - buffered = self.processor.commit_next_scenario() - self.assertEqual(buffered, 1) + self.assertEqual(self.processor.scenarios_committed, 1) + self.assertEqual(self.processor.scenarios_pending, 1) self.processor.next_scenario_request() - buffered = self.processor.commit_next_scenario() - self.assertEqual(buffered, 0) + self.assertEqual(self.processor.scenarios_pending, 0) self.processor.next_scenario_request() - buffered = self.processor.commit_next_scenario() - self.assertEqual(buffered, 1) + self.assertEqual(self.processor.scenarios_pending, 1) self.processor.next_scenario_request() - buffered = self.processor.commit_next_scenario() - self.assertEqual(buffered, 0) + self.assertEqual(self.processor.scenarios_pending, 0) self.assertEqual(out_suite.scenario_count(), 4) def test_batch_size_10(self): out_suite = self.processor.process_test_suite(self.suite, scenario_target=15, batch_size=10) self.processor.next_scenario_request() self.assertEqual(out_suite.scenario_count(), 1) - buffered = self.processor.commit_next_scenario() - self.assertEqual(buffered, 9) + self.assertEqual(self.processor.scenarios_pending, 9) for _ in range(9): self.processor.next_scenario_request() - buffered = self.processor.commit_next_scenario() self.assertEqual(out_suite.scenario_count(), 10) - self.assertEqual(buffered, 0) + self.assertEqual(self.processor.scenarios_pending, 0) self.processor.next_scenario_request() - buffered = self.processor.commit_next_scenario() - self.assertEqual(buffered, 4) + self.assertEqual(self.processor.scenarios_pending, 4) def test_batch_size_3_is_trace_length(self): out_suite = self.processor.process_test_suite(self.suite, scenario_target=3, batch_size=3) + self.assertEqual(self.processor.scenarios_pending, 3) self.processor.next_scenario_request() - buffered = self.processor.commit_next_scenario() - self.assertEqual(buffered, 2) + self.assertEqual(self.processor.scenarios_pending, 2) self.processor.next_scenario_request() self.processor.next_scenario_request() self.assertEqual(out_suite.scenario_count(), 3) @@ -195,12 +191,9 @@ def test_batch_size_3_is_trace_length(self): def test_requesting_beyond_targets_has_no_effect(self): out_suite = self.processor.process_test_suite(self.suite, scenario_target=3, batch_size=3) self.processor.next_scenario_request() - self.processor.commit_next_scenario() self.processor.next_scenario_request() - self.processor.commit_next_scenario() self.assertFalse(self.processor.are_all_targets_reached()) self.processor.next_scenario_request() - self.processor.commit_next_scenario() self.assertEqual(out_suite.scenario_count(), 3) self.assertTrue(self.processor.are_all_targets_reached()) self.processor.next_scenario_request() @@ -219,7 +212,6 @@ def test_multi_batch(self): scenario_target=target, batch_size=batch) while not self.processor.are_all_targets_reached(): self.processor.next_scenario_request() - self.processor.commit_next_scenario() self.assertEqual(out_suite.scenario_count(), target) self.assertListEqual([s.name[:self.scenario_name_without_rep_count] for s in out_suite.scenarios], ['init scenario'] + ['body scenario']*(out_suite.scenario_count()-1)) From adad942ec3c935590212bb495c24fb968661c83f Mon Sep 17 00:00:00 2001 From: JFoederer <32476108+JFoederer@users.noreply.github.com> Date: Wed, 17 Jun 2026 16:16:21 +0200 Subject: [PATCH 14/19] make direct trace discovery respect batch size --- .../01__pass_option_directly.robot | 0 .../02__set_option_by_keyword.robot | 0 .../03__update_option_by_keyword.robot | 0 .../04__update_option_at_trigger.robot | 0 .../05__use_update_without_setter.robot | 0 .../06__pass_multiple_options.robot | 0 .../07__set_clears_other_options.robot | 0 .../08__empty_setter_clears_all_options.robot | 0 .../09__multiple_options_from_dict.robot | 0 .../10__partial_option_update.robot | 0 .../11__argument_restrictions.robot | 0 .../__init__.robot | 0 .../with_bonus_scenario.robot | 0 .../without_bonus_scenario.robot | 0 .../01__with_bonus_scenario_option.robot | 0 ..._without_using_bonus_scenario_option.robot | 0 .../__init__.robot | 0 ...ct_setting_overrules_library_setting.robot | 0 .../04__prior_overrule_does_not_persist.robot | 0 .../__init__.robot | 0 .../strictsuiterepeater.py | 0 .../suiterepeater.py | 0 .../__init__.robot | 0 .../no_seed.robot | 0 .../seed_new.robot | 0 .../seed_none.robot | 0 .../01__generating_random_traces/traces.py | 0 .../02__reusing_seed_reproduces_trace.robot | 0 .../03__retrace_with_refinement.robot | 0 .../04__retrace_with_step_modifiers.robot | 0 .../05__retrace_combined.robot | 0 ...__stop_at_single_coverage_by_default.robot | 0 .../02__stop_beyond_single_coverage.robot | 0 .../03__stop_before_single_coverage.robot | 0 ...st_complete_to_reach_coverage_target.robot | 0 ...tay_incomplete_once_coverage_reached.robot | 0 .../04__batch_size/tag_listener.py | 29 ++++++++++++++++++ .../trace_extension_is_tagged.robot | 30 +++++++++++++++++++ robotmbt/suiteprocessors.py | 26 ++++++++++------ robotmbt/suitereplacer.py | 2 +- robotmbt/visualise/models.py | 2 +- 41 files changed, 78 insertions(+), 11 deletions(-) rename atest/robotMBT tests/07__processor_options/{option_handling => 01__option_handling}/01__pass_option_directly.robot (100%) rename atest/robotMBT tests/07__processor_options/{option_handling => 01__option_handling}/02__set_option_by_keyword.robot (100%) rename atest/robotMBT tests/07__processor_options/{option_handling => 01__option_handling}/03__update_option_by_keyword.robot (100%) rename atest/robotMBT tests/07__processor_options/{option_handling => 01__option_handling}/04__update_option_at_trigger.robot (100%) rename atest/robotMBT tests/07__processor_options/{option_handling => 01__option_handling}/05__use_update_without_setter.robot (100%) rename atest/robotMBT tests/07__processor_options/{option_handling => 01__option_handling}/06__pass_multiple_options.robot (100%) rename atest/robotMBT tests/07__processor_options/{option_handling => 01__option_handling}/07__set_clears_other_options.robot (100%) rename atest/robotMBT tests/07__processor_options/{option_handling => 01__option_handling}/08__empty_setter_clears_all_options.robot (100%) rename atest/robotMBT tests/07__processor_options/{option_handling => 01__option_handling}/09__multiple_options_from_dict.robot (100%) rename atest/robotMBT tests/07__processor_options/{option_handling => 01__option_handling}/10__partial_option_update.robot (100%) rename atest/robotMBT tests/07__processor_options/{option_handling => 01__option_handling}/11__argument_restrictions.robot (100%) rename atest/robotMBT tests/07__processor_options/{option_handling => 01__option_handling}/12__settings_can_span_multiple_suites/__init__.robot (100%) rename atest/robotMBT tests/07__processor_options/{option_handling => 01__option_handling}/12__settings_can_span_multiple_suites/with_bonus_scenario.robot (100%) rename atest/robotMBT tests/07__processor_options/{option_handling => 01__option_handling}/12__settings_can_span_multiple_suites/without_bonus_scenario.robot (100%) rename atest/robotMBT tests/07__processor_options/{option_handling => 01__option_handling}/13__direct_settings_affect_current_suite_only/01__with_bonus_scenario_option.robot (100%) rename atest/robotMBT tests/07__processor_options/{option_handling => 01__option_handling}/13__direct_settings_affect_current_suite_only/02__without_using_bonus_scenario_option.robot (100%) rename atest/robotMBT tests/07__processor_options/{option_handling => 01__option_handling}/13__direct_settings_affect_current_suite_only/__init__.robot (100%) rename atest/robotMBT tests/07__processor_options/{option_handling => 01__option_handling}/14__overruling_library_setting_affects_current_suite_only/03__direct_setting_overrules_library_setting.robot (100%) rename atest/robotMBT tests/07__processor_options/{option_handling => 01__option_handling}/14__overruling_library_setting_affects_current_suite_only/04__prior_overrule_does_not_persist.robot (100%) rename atest/robotMBT tests/07__processor_options/{option_handling => 01__option_handling}/14__overruling_library_setting_affects_current_suite_only/__init__.robot (100%) rename atest/robotMBT tests/07__processor_options/{option_handling => 01__option_handling}/strictsuiterepeater.py (100%) rename atest/robotMBT tests/07__processor_options/{option_handling => 01__option_handling}/suiterepeater.py (100%) rename atest/robotMBT tests/07__processor_options/{random_seeds => 02__random_seeds}/01__generating_random_traces/__init__.robot (100%) rename atest/robotMBT tests/07__processor_options/{random_seeds => 02__random_seeds}/01__generating_random_traces/no_seed.robot (100%) rename atest/robotMBT tests/07__processor_options/{random_seeds => 02__random_seeds}/01__generating_random_traces/seed_new.robot (100%) rename atest/robotMBT tests/07__processor_options/{random_seeds => 02__random_seeds}/01__generating_random_traces/seed_none.robot (100%) rename atest/robotMBT tests/07__processor_options/{random_seeds => 02__random_seeds}/01__generating_random_traces/traces.py (100%) rename atest/robotMBT tests/07__processor_options/{random_seeds => 02__random_seeds}/02__reusing_seed_reproduces_trace.robot (100%) rename atest/robotMBT tests/07__processor_options/{random_seeds => 02__random_seeds}/03__retrace_with_refinement.robot (100%) rename atest/robotMBT tests/07__processor_options/{random_seeds => 02__random_seeds}/04__retrace_with_step_modifiers.robot (100%) rename atest/robotMBT tests/07__processor_options/{random_seeds => 02__random_seeds}/05__retrace_combined.robot (100%) rename atest/robotMBT tests/07__processor_options/{stop_conditions => 03__stop_conditions}/01__stop_at_single_coverage_by_default.robot (100%) rename atest/robotMBT tests/07__processor_options/{stop_conditions => 03__stop_conditions}/02__stop_beyond_single_coverage.robot (100%) rename atest/robotMBT tests/07__processor_options/{stop_conditions => 03__stop_conditions}/03__stop_before_single_coverage.robot (100%) rename atest/robotMBT tests/07__processor_options/{stop_conditions => 03__stop_conditions}/04__refinement_must_complete_to_reach_coverage_target.robot (100%) rename atest/robotMBT tests/07__processor_options/{stop_conditions => 03__stop_conditions}/05__refinement_can_stay_incomplete_once_coverage_reached.robot (100%) create mode 100644 atest/robotMBT tests/07__processor_options/04__batch_size/tag_listener.py create mode 100644 atest/robotMBT tests/07__processor_options/04__batch_size/trace_extension_is_tagged.robot diff --git a/atest/robotMBT tests/07__processor_options/option_handling/01__pass_option_directly.robot b/atest/robotMBT tests/07__processor_options/01__option_handling/01__pass_option_directly.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/option_handling/01__pass_option_directly.robot rename to atest/robotMBT tests/07__processor_options/01__option_handling/01__pass_option_directly.robot diff --git a/atest/robotMBT tests/07__processor_options/option_handling/02__set_option_by_keyword.robot b/atest/robotMBT tests/07__processor_options/01__option_handling/02__set_option_by_keyword.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/option_handling/02__set_option_by_keyword.robot rename to atest/robotMBT tests/07__processor_options/01__option_handling/02__set_option_by_keyword.robot diff --git a/atest/robotMBT tests/07__processor_options/option_handling/03__update_option_by_keyword.robot b/atest/robotMBT tests/07__processor_options/01__option_handling/03__update_option_by_keyword.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/option_handling/03__update_option_by_keyword.robot rename to atest/robotMBT tests/07__processor_options/01__option_handling/03__update_option_by_keyword.robot diff --git a/atest/robotMBT tests/07__processor_options/option_handling/04__update_option_at_trigger.robot b/atest/robotMBT tests/07__processor_options/01__option_handling/04__update_option_at_trigger.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/option_handling/04__update_option_at_trigger.robot rename to atest/robotMBT tests/07__processor_options/01__option_handling/04__update_option_at_trigger.robot diff --git a/atest/robotMBT tests/07__processor_options/option_handling/05__use_update_without_setter.robot b/atest/robotMBT tests/07__processor_options/01__option_handling/05__use_update_without_setter.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/option_handling/05__use_update_without_setter.robot rename to atest/robotMBT tests/07__processor_options/01__option_handling/05__use_update_without_setter.robot diff --git a/atest/robotMBT tests/07__processor_options/option_handling/06__pass_multiple_options.robot b/atest/robotMBT tests/07__processor_options/01__option_handling/06__pass_multiple_options.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/option_handling/06__pass_multiple_options.robot rename to atest/robotMBT tests/07__processor_options/01__option_handling/06__pass_multiple_options.robot diff --git a/atest/robotMBT tests/07__processor_options/option_handling/07__set_clears_other_options.robot b/atest/robotMBT tests/07__processor_options/01__option_handling/07__set_clears_other_options.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/option_handling/07__set_clears_other_options.robot rename to atest/robotMBT tests/07__processor_options/01__option_handling/07__set_clears_other_options.robot diff --git a/atest/robotMBT tests/07__processor_options/option_handling/08__empty_setter_clears_all_options.robot b/atest/robotMBT tests/07__processor_options/01__option_handling/08__empty_setter_clears_all_options.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/option_handling/08__empty_setter_clears_all_options.robot rename to atest/robotMBT tests/07__processor_options/01__option_handling/08__empty_setter_clears_all_options.robot diff --git a/atest/robotMBT tests/07__processor_options/option_handling/09__multiple_options_from_dict.robot b/atest/robotMBT tests/07__processor_options/01__option_handling/09__multiple_options_from_dict.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/option_handling/09__multiple_options_from_dict.robot rename to atest/robotMBT tests/07__processor_options/01__option_handling/09__multiple_options_from_dict.robot diff --git a/atest/robotMBT tests/07__processor_options/option_handling/10__partial_option_update.robot b/atest/robotMBT tests/07__processor_options/01__option_handling/10__partial_option_update.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/option_handling/10__partial_option_update.robot rename to atest/robotMBT tests/07__processor_options/01__option_handling/10__partial_option_update.robot diff --git a/atest/robotMBT tests/07__processor_options/option_handling/11__argument_restrictions.robot b/atest/robotMBT tests/07__processor_options/01__option_handling/11__argument_restrictions.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/option_handling/11__argument_restrictions.robot rename to atest/robotMBT tests/07__processor_options/01__option_handling/11__argument_restrictions.robot diff --git a/atest/robotMBT tests/07__processor_options/option_handling/12__settings_can_span_multiple_suites/__init__.robot b/atest/robotMBT tests/07__processor_options/01__option_handling/12__settings_can_span_multiple_suites/__init__.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/option_handling/12__settings_can_span_multiple_suites/__init__.robot rename to atest/robotMBT tests/07__processor_options/01__option_handling/12__settings_can_span_multiple_suites/__init__.robot diff --git a/atest/robotMBT tests/07__processor_options/option_handling/12__settings_can_span_multiple_suites/with_bonus_scenario.robot b/atest/robotMBT tests/07__processor_options/01__option_handling/12__settings_can_span_multiple_suites/with_bonus_scenario.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/option_handling/12__settings_can_span_multiple_suites/with_bonus_scenario.robot rename to atest/robotMBT tests/07__processor_options/01__option_handling/12__settings_can_span_multiple_suites/with_bonus_scenario.robot diff --git a/atest/robotMBT tests/07__processor_options/option_handling/12__settings_can_span_multiple_suites/without_bonus_scenario.robot b/atest/robotMBT tests/07__processor_options/01__option_handling/12__settings_can_span_multiple_suites/without_bonus_scenario.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/option_handling/12__settings_can_span_multiple_suites/without_bonus_scenario.robot rename to atest/robotMBT tests/07__processor_options/01__option_handling/12__settings_can_span_multiple_suites/without_bonus_scenario.robot diff --git a/atest/robotMBT tests/07__processor_options/option_handling/13__direct_settings_affect_current_suite_only/01__with_bonus_scenario_option.robot b/atest/robotMBT tests/07__processor_options/01__option_handling/13__direct_settings_affect_current_suite_only/01__with_bonus_scenario_option.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/option_handling/13__direct_settings_affect_current_suite_only/01__with_bonus_scenario_option.robot rename to atest/robotMBT tests/07__processor_options/01__option_handling/13__direct_settings_affect_current_suite_only/01__with_bonus_scenario_option.robot diff --git a/atest/robotMBT tests/07__processor_options/option_handling/13__direct_settings_affect_current_suite_only/02__without_using_bonus_scenario_option.robot b/atest/robotMBT tests/07__processor_options/01__option_handling/13__direct_settings_affect_current_suite_only/02__without_using_bonus_scenario_option.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/option_handling/13__direct_settings_affect_current_suite_only/02__without_using_bonus_scenario_option.robot rename to atest/robotMBT tests/07__processor_options/01__option_handling/13__direct_settings_affect_current_suite_only/02__without_using_bonus_scenario_option.robot diff --git a/atest/robotMBT tests/07__processor_options/option_handling/13__direct_settings_affect_current_suite_only/__init__.robot b/atest/robotMBT tests/07__processor_options/01__option_handling/13__direct_settings_affect_current_suite_only/__init__.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/option_handling/13__direct_settings_affect_current_suite_only/__init__.robot rename to atest/robotMBT tests/07__processor_options/01__option_handling/13__direct_settings_affect_current_suite_only/__init__.robot diff --git a/atest/robotMBT tests/07__processor_options/option_handling/14__overruling_library_setting_affects_current_suite_only/03__direct_setting_overrules_library_setting.robot b/atest/robotMBT tests/07__processor_options/01__option_handling/14__overruling_library_setting_affects_current_suite_only/03__direct_setting_overrules_library_setting.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/option_handling/14__overruling_library_setting_affects_current_suite_only/03__direct_setting_overrules_library_setting.robot rename to atest/robotMBT tests/07__processor_options/01__option_handling/14__overruling_library_setting_affects_current_suite_only/03__direct_setting_overrules_library_setting.robot diff --git a/atest/robotMBT tests/07__processor_options/option_handling/14__overruling_library_setting_affects_current_suite_only/04__prior_overrule_does_not_persist.robot b/atest/robotMBT tests/07__processor_options/01__option_handling/14__overruling_library_setting_affects_current_suite_only/04__prior_overrule_does_not_persist.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/option_handling/14__overruling_library_setting_affects_current_suite_only/04__prior_overrule_does_not_persist.robot rename to atest/robotMBT tests/07__processor_options/01__option_handling/14__overruling_library_setting_affects_current_suite_only/04__prior_overrule_does_not_persist.robot diff --git a/atest/robotMBT tests/07__processor_options/option_handling/14__overruling_library_setting_affects_current_suite_only/__init__.robot b/atest/robotMBT tests/07__processor_options/01__option_handling/14__overruling_library_setting_affects_current_suite_only/__init__.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/option_handling/14__overruling_library_setting_affects_current_suite_only/__init__.robot rename to atest/robotMBT tests/07__processor_options/01__option_handling/14__overruling_library_setting_affects_current_suite_only/__init__.robot diff --git a/atest/robotMBT tests/07__processor_options/option_handling/strictsuiterepeater.py b/atest/robotMBT tests/07__processor_options/01__option_handling/strictsuiterepeater.py similarity index 100% rename from atest/robotMBT tests/07__processor_options/option_handling/strictsuiterepeater.py rename to atest/robotMBT tests/07__processor_options/01__option_handling/strictsuiterepeater.py diff --git a/atest/robotMBT tests/07__processor_options/option_handling/suiterepeater.py b/atest/robotMBT tests/07__processor_options/01__option_handling/suiterepeater.py similarity index 100% rename from atest/robotMBT tests/07__processor_options/option_handling/suiterepeater.py rename to atest/robotMBT tests/07__processor_options/01__option_handling/suiterepeater.py diff --git a/atest/robotMBT tests/07__processor_options/random_seeds/01__generating_random_traces/__init__.robot b/atest/robotMBT tests/07__processor_options/02__random_seeds/01__generating_random_traces/__init__.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/random_seeds/01__generating_random_traces/__init__.robot rename to atest/robotMBT tests/07__processor_options/02__random_seeds/01__generating_random_traces/__init__.robot diff --git a/atest/robotMBT tests/07__processor_options/random_seeds/01__generating_random_traces/no_seed.robot b/atest/robotMBT tests/07__processor_options/02__random_seeds/01__generating_random_traces/no_seed.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/random_seeds/01__generating_random_traces/no_seed.robot rename to atest/robotMBT tests/07__processor_options/02__random_seeds/01__generating_random_traces/no_seed.robot diff --git a/atest/robotMBT tests/07__processor_options/random_seeds/01__generating_random_traces/seed_new.robot b/atest/robotMBT tests/07__processor_options/02__random_seeds/01__generating_random_traces/seed_new.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/random_seeds/01__generating_random_traces/seed_new.robot rename to atest/robotMBT tests/07__processor_options/02__random_seeds/01__generating_random_traces/seed_new.robot diff --git a/atest/robotMBT tests/07__processor_options/random_seeds/01__generating_random_traces/seed_none.robot b/atest/robotMBT tests/07__processor_options/02__random_seeds/01__generating_random_traces/seed_none.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/random_seeds/01__generating_random_traces/seed_none.robot rename to atest/robotMBT tests/07__processor_options/02__random_seeds/01__generating_random_traces/seed_none.robot diff --git a/atest/robotMBT tests/07__processor_options/random_seeds/01__generating_random_traces/traces.py b/atest/robotMBT tests/07__processor_options/02__random_seeds/01__generating_random_traces/traces.py similarity index 100% rename from atest/robotMBT tests/07__processor_options/random_seeds/01__generating_random_traces/traces.py rename to atest/robotMBT tests/07__processor_options/02__random_seeds/01__generating_random_traces/traces.py diff --git a/atest/robotMBT tests/07__processor_options/random_seeds/02__reusing_seed_reproduces_trace.robot b/atest/robotMBT tests/07__processor_options/02__random_seeds/02__reusing_seed_reproduces_trace.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/random_seeds/02__reusing_seed_reproduces_trace.robot rename to atest/robotMBT tests/07__processor_options/02__random_seeds/02__reusing_seed_reproduces_trace.robot diff --git a/atest/robotMBT tests/07__processor_options/random_seeds/03__retrace_with_refinement.robot b/atest/robotMBT tests/07__processor_options/02__random_seeds/03__retrace_with_refinement.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/random_seeds/03__retrace_with_refinement.robot rename to atest/robotMBT tests/07__processor_options/02__random_seeds/03__retrace_with_refinement.robot diff --git a/atest/robotMBT tests/07__processor_options/random_seeds/04__retrace_with_step_modifiers.robot b/atest/robotMBT tests/07__processor_options/02__random_seeds/04__retrace_with_step_modifiers.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/random_seeds/04__retrace_with_step_modifiers.robot rename to atest/robotMBT tests/07__processor_options/02__random_seeds/04__retrace_with_step_modifiers.robot diff --git a/atest/robotMBT tests/07__processor_options/random_seeds/05__retrace_combined.robot b/atest/robotMBT tests/07__processor_options/02__random_seeds/05__retrace_combined.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/random_seeds/05__retrace_combined.robot rename to atest/robotMBT tests/07__processor_options/02__random_seeds/05__retrace_combined.robot diff --git a/atest/robotMBT tests/07__processor_options/stop_conditions/01__stop_at_single_coverage_by_default.robot b/atest/robotMBT tests/07__processor_options/03__stop_conditions/01__stop_at_single_coverage_by_default.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/stop_conditions/01__stop_at_single_coverage_by_default.robot rename to atest/robotMBT tests/07__processor_options/03__stop_conditions/01__stop_at_single_coverage_by_default.robot diff --git a/atest/robotMBT tests/07__processor_options/stop_conditions/02__stop_beyond_single_coverage.robot b/atest/robotMBT tests/07__processor_options/03__stop_conditions/02__stop_beyond_single_coverage.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/stop_conditions/02__stop_beyond_single_coverage.robot rename to atest/robotMBT tests/07__processor_options/03__stop_conditions/02__stop_beyond_single_coverage.robot diff --git a/atest/robotMBT tests/07__processor_options/stop_conditions/03__stop_before_single_coverage.robot b/atest/robotMBT tests/07__processor_options/03__stop_conditions/03__stop_before_single_coverage.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/stop_conditions/03__stop_before_single_coverage.robot rename to atest/robotMBT tests/07__processor_options/03__stop_conditions/03__stop_before_single_coverage.robot diff --git a/atest/robotMBT tests/07__processor_options/stop_conditions/04__refinement_must_complete_to_reach_coverage_target.robot b/atest/robotMBT tests/07__processor_options/03__stop_conditions/04__refinement_must_complete_to_reach_coverage_target.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/stop_conditions/04__refinement_must_complete_to_reach_coverage_target.robot rename to atest/robotMBT tests/07__processor_options/03__stop_conditions/04__refinement_must_complete_to_reach_coverage_target.robot diff --git a/atest/robotMBT tests/07__processor_options/stop_conditions/05__refinement_can_stay_incomplete_once_coverage_reached.robot b/atest/robotMBT tests/07__processor_options/03__stop_conditions/05__refinement_can_stay_incomplete_once_coverage_reached.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/stop_conditions/05__refinement_can_stay_incomplete_once_coverage_reached.robot rename to atest/robotMBT tests/07__processor_options/03__stop_conditions/05__refinement_can_stay_incomplete_once_coverage_reached.robot diff --git a/atest/robotMBT tests/07__processor_options/04__batch_size/tag_listener.py b/atest/robotMBT tests/07__processor_options/04__batch_size/tag_listener.py new file mode 100644 index 0000000..373b3f9 --- /dev/null +++ b/atest/robotMBT tests/07__processor_options/04__batch_size/tag_listener.py @@ -0,0 +1,29 @@ +from robot.api import logger +from robot.api.deco import library +from robot.libraries.BuiltIn import BuiltIn + + +@library(scope='SUITE', listener='SELF') +class TagListener: + ROBOT_LISTENER_PRIORITY = -1 # Set lower priority to make sure tags are already set by robotmbt + TRACE_TAG = 'mbt trace extension' + + def __init__(self): + self.test_count = 0 + + def end_test(self, tc, result): + self.test_count += 1 + if self.test_count % 2: + if self.TRACE_TAG in result.tags: + result.status = 'FAIL' + result.message = f"Unexpected test tag '{self.TRACE_TAG}'" + else: + if self.TRACE_TAG not in result.tags: + result.status = 'FAIL' + result.message = f"Test tag '{self.TRACE_TAG}' missing" + + if 'my tag' not in result.tags: + result.status = 'FAIL' + result.message = "Test tag 'my tag' missing" + BuiltIn().set_suite_variable('${confirmed_passes}', BuiltIn().get_variable_value('${confirmed_passes}') + 1) + logger.info("PASS confirmed by listener") diff --git a/atest/robotMBT tests/07__processor_options/04__batch_size/trace_extension_is_tagged.robot b/atest/robotMBT tests/07__processor_options/04__batch_size/trace_extension_is_tagged.robot new file mode 100644 index 0000000..8b7a1eb --- /dev/null +++ b/atest/robotMBT tests/07__processor_options/04__batch_size/trace_extension_is_tagged.robot @@ -0,0 +1,30 @@ +*** Settings *** +Documentation This test suite checks that Batch size for trace generation is respected and that +... the (logging for) trace generation can be found, even when it is not part of the +... initial 'Treat this test suite model-based' keyword. To find the delayed parts of +... trace generation, the scenarios that trigger trace extension are tagged. Since +... tagging is not done until `end_test`, a listener is used to check and confirm +... that the expected scenarios are tagged. +... +... Note that tagging is not the preferred solution, but alternatives failed due to +... Robot Framework's scoping limitations. +Suite Setup Treat this test suite Model-based batch_size=2 +Suite Teardown Should Be Equal ${confirmed_passes} ${3} +Test Tags my tag +Library robotmbt +Library tag_listener.py + + +*** Variables *** +${confirmed_passes} ${0} + + +*** Test Cases *** +Scenario 1 + No Operation + +Scenario 2 + No Operation + +Scenario 3 + No Operation diff --git a/robotmbt/suiteprocessors.py b/robotmbt/suiteprocessors.py index fab8817..9e5a5ea 100644 --- a/robotmbt/suiteprocessors.py +++ b/robotmbt/suiteprocessors.py @@ -172,10 +172,15 @@ def process_test_suite(self, in_suite: Suite, *, seed: str | int | bytes | bytea try: # a short trace without the need for repeating scenarios is preferred direct_tracestate = self._search_direct_trace() - if self.are_all_targets_reached(direct_tracestate, committed_only=False): - # The visualiser assumes that the last trace is the final selected trace, which is not always - # the case. Re-adding the selected trace to prevent the wrong path from being highlighted. + if self._discovery_ready(direct_tracestate): self.tracestate = direct_tracestate + n = len(direct_tracestate.covered_ids) + logger.debug(f"Using discovered trace ({n} scenario{'s' if n != 1 else ''}):" + f" [{', '.join(direct_tracestate.id_trace)}]") + # The visualiser assumes that the last trace is the final selected trace, which is not always + # the case. Re-initialising and then adding the selected trace again prevents the wrong path + # from being highlighted. + self._update_visualisation(TraceState(direct_tracestate.prio_order)) self._update_visualisation(self.tracestate) else: self.tracestate = TraceState([s.src_id for s in self.scenarios]) @@ -261,23 +266,23 @@ def _search_direct_trace(self) -> TraceState: if self._is_duplicate_prio_order(tracestates, prio_order): continue tracestates.append(self._one_shot_trace(prio_order)) - if self.are_all_targets_reached(tracestates[-1], committed_only=False) and not self._visualiser: + if self._discovery_ready(tracestates[-1]) and not self._visualiser: return tracestates[-1] suggestion = self._create_suggestion_by_experience(tracestates) if self._is_duplicate_prio_order(tracestates, suggestion): continue tracestates.append(self._one_shot_trace(suggestion)) - if self.are_all_targets_reached(tracestates[-1], committed_only=False) and not self._visualiser: + if self._discovery_ready(tracestates[-1]) and not self._visualiser: return tracestates[-1] index_longest = self._longest_trace(tracestates) - if self.are_all_targets_reached(tracestates[index_longest], committed_only=False): + if self._discovery_ready(tracestates[index_longest]): return tracestates[index_longest] logger.debug("Trying to extend most promising traces") prio_order = self._create_suggestion_by_experience(tracestates, index_longest) if not self._is_duplicate_prio_order(tracestates, prio_order): tracestates.append(self._one_shot_trace(prio_order)) - if self.are_all_targets_reached(tracestates[-1], committed_only=False): + if self._discovery_ready(tracestates[-1]): return tracestates[-1] last_new = self._last_new_coverage(tracestates) while True: # while still discovering new coverage @@ -285,7 +290,7 @@ def _search_direct_trace(self) -> TraceState: if self._is_duplicate_prio_order(tracestates, prio_order): break tracestates.append(self._one_shot_trace(prio_order)) - if self.are_all_targets_reached(tracestates[-1], committed_only=False): + if self._discovery_ready(tracestates[-1]): return tracestates[-1] last_new = self._last_new_coverage(tracestates) if last_new != len(tracestates)-1: @@ -302,6 +307,9 @@ def _search_direct_trace(self) -> TraceState: "(Scenarios marked with * are not part of any trace)\n\n") return longest + def _discovery_ready(self, tracestate): + return self.are_all_targets_reached(tracestate, committed_only=False) or len(tracestate) >= self.batch_size + def _longest_trace(self, tracestate_list: list[TraceState]) -> int: """returns the index of the trace that covers the most scenarios""" lengths = [len(ts.covered_ids) for ts in tracestate_list] @@ -341,7 +349,7 @@ def _one_shot_trace(self, scenarios: list[int]) -> TraceState: tracestate = TraceState(scenarios) self._update_visualisation(tracestate) candidate_id = tracestate.next_candidate(retry=False, randomise=False) - while candidate_id is not None: + while candidate_id is not None and not self._discovery_ready(tracestate): candidate = self._select_scenario_variant(candidate_id, tracestate) if candidate: # No valid variant available in the current state modeller.try_to_fit_in_scenario(candidate, tracestate) diff --git a/robotmbt/suitereplacer.py b/robotmbt/suitereplacer.py index 942c34d..8fd03b1 100644 --- a/robotmbt/suitereplacer.py +++ b/robotmbt/suitereplacer.py @@ -251,7 +251,7 @@ def _end_test(self, test_case: rmodel.TestCase, result: robot.result.model.TestC old_total = committed_old + pending_old if new_total > old_total: result.tags.add('mbt trace extension') - logger.info(f"MBT trace generation added {new_total-old_total} new scenarios.") + logger.info(f"MBT trace generation prepared {new_total-old_total} new scenarios.") if not pending_old and new_total == old_total: logger.info(f"Trace could not be extended.") try: diff --git a/robotmbt/visualise/models.py b/robotmbt/visualise/models.py index 658d49b..f30c7cc 100644 --- a/robotmbt/visualise/models.py +++ b/robotmbt/visualise/models.py @@ -230,7 +230,7 @@ def update_trace(self, scenario: ScenarioInfo | None, state: StateInfo, length: else: # No change - sanity check if len(self.current_trace) > 0: - self._sanity_check(scenario, state, 'nothing') + self._sanity_check(scenario, state, 'nothing changed') def _push(self, scenario: ScenarioInfo, state: StateInfo, n: int): if n > 1: From c19e99fd33dd06188141e3a5c329de2e887dac90 Mon Sep 17 00:00:00 2001 From: JFoederer <32476108+JFoederer@users.noreply.github.com> Date: Wed, 17 Jun 2026 18:16:34 +0200 Subject: [PATCH 15/19] basic fail when not all targets are achieved --- robotmbt/suitereplacer.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/robotmbt/suitereplacer.py b/robotmbt/suitereplacer.py index 8fd03b1..843696d 100644 --- a/robotmbt/suitereplacer.py +++ b/robotmbt/suitereplacer.py @@ -218,10 +218,10 @@ def add_test(tc: Scenario, target_suite: robot.model.TestSuite): new_tc.body.create_keyword(name=step.keyword, assign=step.assign, args=step.posnom_args_str) target_suite.tests.append(new_tc) - def _start_suite(self, suite: rmodel.TestSuite, result): + def _start_suite(self, suite: rmodel.TestSuite, result: robot.result.model.TestSuite): self.current_suite = suite - def _end_suite(self, suite: rmodel.TestSuite, result): + def _end_suite(self, suite: rmodel.TestSuite, result: robot.result.model.TestSuite): if suite == self.mbt_anchor_suite: self.mbt_anchor_suite = None if not self.mbt_anchor_suite: @@ -243,17 +243,23 @@ def _end_test(self, test_case: rmodel.TestCase, result: robot.result.model.TestC committed_old = self.processor.scenarios_committed pending_old = self.processor.scenarios_pending if not pending_old: - logger.info(f"{committed_old} scenarios completed. Looking to extend trace.") + logger.info(f"{committed_old} Scenario{'s' if committed_old != 1 else ''} completed. Looking to extend trace.") self.processor.next_scenario_request() committed = self.processor.scenarios_committed pending = self.processor.scenarios_pending new_total = committed + pending old_total = committed_old + pending_old + if not pending_old and new_total == old_total: + logger.info(f"Trace could not be extended.") + if not self.processor.are_all_targets_reached(): + new_tc = self.current_suite.tests.create(name='Confirm exit criteria') + new_tc.body.create_keyword(name='Fail', args=('Not all targets achieved',)) + self.mbt_anchor_suite = None + return + if new_total > old_total: result.tags.add('mbt trace extension') logger.info(f"MBT trace generation prepared {new_total-old_total} new scenarios.") - if not pending_old and new_total == old_total: - logger.info(f"Trace could not be extended.") try: self.add_test(next(self.test_case_gen[-1]), self.current_suite) except StopIteration: From 0f9f88645fb89a8f13cd70f352b4215bd5258f4a Mon Sep 17 00:00:00 2001 From: JFoederer <32476108+JFoederer@users.noreply.github.com> Date: Fri, 19 Jun 2026 11:25:06 +0200 Subject: [PATCH 16/19] document run targets and batch sizes --- README.md | 45 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 71f25a6..b09f09f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # RobotMBT - the oneliner - Model-based testing in Robot framework with test case generation + Model-based testing in Robot framework with dynamic trace and test case generation ## Introduction @@ -24,7 +24,7 @@ RobotMBT offers features to cover both _when_ and _what_ variations. RobotMBT is suitable for sequencing complete scenarios, including action refinement for when-steps. Concrete example scenarios can be generalised for added data-driven variation. When all steps are properly annotated with modelling info, the library can resolve their dependencies and figure out the correct execution order. Each run a new test sequence is generated from the available options. -To be successful, the set of scenarios in the model must (for now) be composable into a single complete sequence, without leftovers. The same scenario can be inserted into the trace multiple times, creating loops, if repetition helps to reach the entry condition for later scenarios. Dead ends should be prevented, i.e., sequences from which there is no way forward and no way to loop back. +To be successful, the set of scenarios in the model must be composable into a single complete sequence. There are no automatic resets or retries. The same scenario can be inserted into the trace multiple times, creating loops. Either to reach otherwise unreachable scenarios, or simply to create longer test runs. Dead ends should be prevented, i.e., sequences from which there is no way forward and no way to loop back. ## Getting started @@ -207,6 +207,33 @@ Modified example values do not cascade. If a modifier expression references anot ## Configuration options +Configure your test run by using any of the options from the list. + +| option | purpose | values (default marked with *) | +|-----------------------------------------|----------------------------------|--------------------------------| +| [coverage_target](#setting-run-targets) | Each scenario must be executed at least this many times | 0 or 1* | +| [scenario_target](#setting-run-targets) | The trace must have at least this many scenarios | 0* or higher | +| [seed](#random-seed) | Re-running a prior trace | a specific seed, new* or None | +| [batch_size](#batch-size) | Phased trace generation | 1 or higher (default 100*) | +| [graph](#graphs) | Visualising the model | None*, scenario or scenario-delta-value | +| [export_graph_data](#exporting-and-importing-graph-data) | Storing graphs as json data | None* or file path | + +Options are available as named arguments: + +```robotframework +Treat this test suite model-based coverage_target=1 scenario_target=250 +``` + +If you want to set configuration options for use in multiple test suites without having to repeat them, the keywords __Set model-based options__ and __Update model-based options__ can be used to configure RobotMBT library options. _Set_ takes the provided options and discards any previously set options. _Update_ allows you to modify existing options or add new ones. Reset all options by calling _Set_ without arguments. Direct options provided to __Treat this test suite model-based__ take precedence over library options and affect only the current test suite. + +Tip: [Robot dictionaries](https://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#dictionary-variable) (`&{ }`) can be used to group related options and pass them as one set. + +### Setting run targets + +By default a trace will be generated that runs to _single coverage_, meaning that each scenario must be included in the trace at least once. This is equivalent to setting `coverage_target=1`. To continue generating longer traces after single coverage is reached, a scenario target can be added. For example, `scenario_target=500` will continue to run until there are 500 scenarios in the trace. Note that when scenarios are split up due to when-step refinement, that each part will count as one scenario. + +If you do not need guaranteed coverage, then `coverage_target=0` will disable the coverage check. Test runs can now finish before all scenarios are executed, once the `scenario_target` is reached. This does not affect the trace generation process, which will still prefer new coverage over repetition. + ### Random seed By default, trace generation is random. The random seed used for the trace is logged by _Treat this test suite model-based_. This seed can be used to rerun the same trace, if no external random factors influence the test run. To activate the seed, pass it as argument: @@ -217,6 +244,14 @@ Treat this test suite model-based seed=eag-etou-cxi-leamv-jsi Using `seed=new` will force generation of a new reusable seed and is identical to omitting the seed argument. To completely bypass seed generation and use the system's random source, use `seed=None`. This has even more variation but does not produce a reusable seed. +### Batch size + +Trace generation is done in _Batches_, so that the test run can already start before the full trace is generated. Small batch sizes cause the test run to start quickly, wheras larger batch sizes give more room to find suitable traces. Batch size is configurable by setting `batch_size=`. + +If the batch size is large enough to reach all run targets in a single batch, then the run wil finish without further extensions. If not all targets are achieved in the first batch, then trace generation continues whenever new scenarios are needed. This is always at the end of a scenario. This scenario is tagged `mbt trace extension` and also contains the logging for the extended trace generation. + +Tip: _Small batch sizes are good at exposing dead ends in your model._ + ### Graphs A graph can be included in the log file to visualise how scenarios are linked. This helps in understanding a test suite's structure and reveals alternative paths that did not make it into the final trace. @@ -252,12 +287,6 @@ Show model graph from exported file json_file_path= graph_style This will draw a graph from the exported file, without the need to rerun the test suite. It is possible to select a different graph style than was used during the test run. If no graph style is selected, then the scenario graph style is used. -### Option management - -If you want to set configuration options for use in multiple test suites without having to repeat them, the keywords __Set model-based options__ and __Update model-based options__ can be used to configure RobotMBT library options. _Set_ takes the provided options and discards any previously set options. _Update_ allows you to modify existing options or add new ones. Reset all options by calling _Set_ without arguments. Direct options provided to __Treat this test suite model-based__ take precedence over library options and affect only the current test suite. - -Tip: [Robot dictionaries](https://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#dictionary-variable) (`&{ }`) can be used to group related options and pass them as one set. - ## Contributing If you have feedback, ideas, or want to get involved in coding, then check out the [Contribution guidelines](https://github.com/JFoederer/robotframeworkMBT/blob/main/CONTRIBUTING.md). From 85250912ea1809d29cdf937ad476312bb9534d7c Mon Sep 17 00:00:00 2001 From: JFoederer <32476108+JFoederer@users.noreply.github.com> Date: Fri, 19 Jun 2026 13:13:30 +0200 Subject: [PATCH 17/19] User reporting improvements --- README.md | 8 +++++--- robotmbt/modeller.py | 6 +++++- robotmbt/suiteprocessors.py | 12 ++++++++---- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index b09f09f..f85485e 100644 --- a/README.md +++ b/README.md @@ -230,10 +230,12 @@ Tip: [Robot dictionaries](https://robotframework.org/robotframework/latest/Robot ### Setting run targets -By default a trace will be generated that runs to _single coverage_, meaning that each scenario must be included in the trace at least once. This is equivalent to setting `coverage_target=1`. To continue generating longer traces after single coverage is reached, a scenario target can be added. For example, `scenario_target=500` will continue to run until there are 500 scenarios in the trace. Note that when scenarios are split up due to when-step refinement, that each part will count as one scenario. +By default, a trace will be generated that runs to _single coverage_, meaning that each scenario must be included in the trace at least once. This is equivalent to setting `coverage_target=1`. To continue generating longer traces after single coverage is reached, a scenario target can be added. For example, `scenario_target=500` will continue to run until there are 500 scenarios in the trace. Note that when scenarios are split up due to when-step refinement, that each part will count as one scenario. If you do not need guaranteed coverage, then `coverage_target=0` will disable the coverage check. Test runs can now finish before all scenarios are executed, once the `scenario_target` is reached. This does not affect the trace generation process, which will still prefer new coverage over repetition. +Tip: _When generating large test suites, use Robot Framework's [Split log](https://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#splitting-logs) feature (`--splitlog`), to keep log file sizes manageable._ + ### Random seed By default, trace generation is random. The random seed used for the trace is logged by _Treat this test suite model-based_. This seed can be used to rerun the same trace, if no external random factors influence the test run. To activate the seed, pass it as argument: @@ -246,9 +248,9 @@ Using `seed=new` will force generation of a new reusable seed and is identical t ### Batch size -Trace generation is done in _Batches_, so that the test run can already start before the full trace is generated. Small batch sizes cause the test run to start quickly, wheras larger batch sizes give more room to find suitable traces. Batch size is configurable by setting `batch_size=`. +Trace generation is done in _Batches_, so that the test run can already start before the full trace is generated. Small batch sizes cause the test run to start quickly, whereas larger batch sizes give more room to find suitable traces. Batch size is configurable by setting `batch_size=`. -If the batch size is large enough to reach all run targets in a single batch, then the run wil finish without further extensions. If not all targets are achieved in the first batch, then trace generation continues whenever new scenarios are needed. This is always at the end of a scenario. This scenario is tagged `mbt trace extension` and also contains the logging for the extended trace generation. +If the batch size is large enough to reach all run targets in a single batch, then the run will finish without further extensions. If not all targets are achieved in the first batch, then trace generation continues whenever new scenarios are needed. This is always at the end of a scenario. This scenario is tagged `mbt trace extension` and also contains the logging for the extended trace generation. Tip: _Small batch sizes are good at exposing dead ends in your model._ diff --git a/robotmbt/modeller.py b/robotmbt/modeller.py index 359a665..901c4ca 100644 --- a/robotmbt/modeller.py +++ b/robotmbt/modeller.py @@ -257,6 +257,10 @@ def _parse_modifier_expression(expression: str, args: StepArguments) -> tuple[st def rewind(tracestate: TraceState, drought_recovery: bool = False) -> TraceSnapShot | None: tail = tracestate.rewind() - while drought_recovery and tracestate.coverage_drought and tracestate.can_rewind(): + while drought_recovery and tracestate.coverage_drought: + if not tracestate.can_rewind(): + logger.debug( + f"Coverage drought recovery stalled. {tracestate.coverage_drought} Scenarios are already committed.") + break tail = tracestate.rewind() return tail diff --git a/robotmbt/suiteprocessors.py b/robotmbt/suiteprocessors.py index 9e5a5ea..f70db68 100644 --- a/robotmbt/suiteprocessors.py +++ b/robotmbt/suiteprocessors.py @@ -175,8 +175,8 @@ def process_test_suite(self, in_suite: Suite, *, seed: str | int | bytes | bytea if self._discovery_ready(direct_tracestate): self.tracestate = direct_tracestate n = len(direct_tracestate.covered_ids) - logger.debug(f"Using discovered trace ({n} scenario{'s' if n != 1 else ''}):" - f" [{', '.join(direct_tracestate.id_trace)}]") + logger.debug(f"Using one of the discovered traces ({n} scenario{'s' if n != 1 else ''})") + self._report_tracestate_to_user(direct_tracestate) # The visualiser assumes that the last trace is the final selected trace, which is not always # the case. Re-initialising and then adding the selected trace again prevents the wrong path # from being highlighted. @@ -267,16 +267,19 @@ def _search_direct_trace(self) -> TraceState: continue tracestates.append(self._one_shot_trace(prio_order)) if self._discovery_ready(tracestates[-1]) and not self._visualiser: + tracestates[-1].unreached = self._unreached_scenarios(tracestates) return tracestates[-1] suggestion = self._create_suggestion_by_experience(tracestates) if self._is_duplicate_prio_order(tracestates, suggestion): continue tracestates.append(self._one_shot_trace(suggestion)) if self._discovery_ready(tracestates[-1]) and not self._visualiser: + tracestates[-1].unreached = self._unreached_scenarios(tracestates) return tracestates[-1] index_longest = self._longest_trace(tracestates) if self._discovery_ready(tracestates[index_longest]): + tracestates[index_longest].unreached = self._unreached_scenarios(tracestates) return tracestates[index_longest] logger.debug("Trying to extend most promising traces") prio_order = self._create_suggestion_by_experience(tracestates, index_longest) @@ -298,7 +301,7 @@ def _search_direct_trace(self) -> TraceState: longest = tracestates[self._longest_trace(tracestates)] not_in_trace = sorted(longest.not_in_trace) - longest.unreached = sorted(self._unreached_scenarios(tracestates)) + longest.unreached = self._unreached_scenarios(tracestates) logger.debug( f"Longest trace so far ({len(longest.covered_ids)} scenario{'s' if len(longest.covered_ids) != 1 else ''})" f": [{', '.join(longest.id_trace)}]\n" @@ -453,7 +456,8 @@ def _fail_on_step_errors(suite: Suite): @staticmethod def _report_tracestate_to_user(tracestate: TraceState): - pending = ', '.join([str(i) + '*' if i in tracestate.unreached else str(i) for i in tracestate.not_in_trace]) + pending = ', '.join([str(i) + '*' if i in tracestate.unreached else str(i) + for i in sorted(tracestate.not_in_trace)]) logger.debug(f"Trace: [{', '.join(tracestate.id_trace)}] Pending: [{pending}]" f"{' Rejected: ' + str(tracestate.tried) if tracestate.tried else ''}") From 9d83c1a87f148d33ff2ff8ffba9445637b23509f Mon Sep 17 00:00:00 2001 From: JFoederer <32476108+JFoederer@users.noreply.github.com> Date: Fri, 19 Jun 2026 15:52:14 +0200 Subject: [PATCH 18/19] add target_time run target --- README.md | 5 +-- .../06__time_target_can_stop_test_run.robot | 33 +++++++++++++++++++ robotmbt/suiteprocessors.py | 21 ++++++++++-- 3 files changed, 54 insertions(+), 5 deletions(-) create mode 100644 atest/robotMBT tests/07__processor_options/03__stop_conditions/06__time_target_can_stop_test_run.robot diff --git a/README.md b/README.md index f85485e..d84fda1 100644 --- a/README.md +++ b/README.md @@ -213,6 +213,7 @@ Configure your test run by using any of the options from the list. |-----------------------------------------|----------------------------------|--------------------------------| | [coverage_target](#setting-run-targets) | Each scenario must be executed at least this many times | 0 or 1* | | [scenario_target](#setting-run-targets) | The trace must have at least this many scenarios | 0* or higher | +| [time_target](#setting-run-targets) | Setting a minimum test run duration | Robot time string | | [seed](#random-seed) | Re-running a prior trace | a specific seed, new* or None | | [batch_size](#batch-size) | Phased trace generation | 1 or higher (default 100*) | | [graph](#graphs) | Visualising the model | None*, scenario or scenario-delta-value | @@ -230,9 +231,9 @@ Tip: [Robot dictionaries](https://robotframework.org/robotframework/latest/Robot ### Setting run targets -By default, a trace will be generated that runs to _single coverage_, meaning that each scenario must be included in the trace at least once. This is equivalent to setting `coverage_target=1`. To continue generating longer traces after single coverage is reached, a scenario target can be added. For example, `scenario_target=500` will continue to run until there are 500 scenarios in the trace. Note that when scenarios are split up due to when-step refinement, that each part will count as one scenario. +By default, a trace will be generated that runs to _single coverage_, meaning that each scenario must be included in the trace at least once. This is equivalent to setting `coverage_target=1`. To continue generating longer traces after single coverage is achieved, a second target can be enabled. For example, `scenario_target=500` will continue to run until there are 500 scenarios in the trace. Note that when scenarios are split up due to when-step refinement, that each part will count as one scenario. Alternatively, setting `time_target=1 hour 30 min` will cause the test run to run at least this long. Refer to the Robot Framework documentation to find [supported time formats](https://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#time-as-time-string). -If you do not need guaranteed coverage, then `coverage_target=0` will disable the coverage check. Test runs can now finish before all scenarios are executed, once the `scenario_target` is reached. This does not affect the trace generation process, which will still prefer new coverage over repetition. +The test run will finish once all enabled targets are achieved. By default, only the coverage target is set. If you do not need guaranteed coverage, then `coverage_target=0` will disable the coverage check. Test runs can now finish before all scenarios are executed. This does not affect the trace generation process, which will still prefer new coverage over repetition. Tip: _When generating large test suites, use Robot Framework's [Split log](https://robotframework.org/robotframework/latest/RobotFrameworkUserGuide.html#splitting-logs) feature (`--splitlog`), to keep log file sizes manageable._ diff --git a/atest/robotMBT tests/07__processor_options/03__stop_conditions/06__time_target_can_stop_test_run.robot b/atest/robotMBT tests/07__processor_options/03__stop_conditions/06__time_target_can_stop_test_run.robot new file mode 100644 index 0000000..8b2b106 --- /dev/null +++ b/atest/robotMBT tests/07__processor_options/03__stop_conditions/06__time_target_can_stop_test_run.robot @@ -0,0 +1,33 @@ +*** Settings *** +Documentation This test suite confirms that a time target can stop a test run if it is the +... last condition to be satisfied. For run duration reasons an unrealistically +... short time span is used, while all other conditions are disabled. This has a +... double effect. One effect is that the test run should always stop after the +... first test case, even though a longer trace was created. We know that a longer +... trace was created due to the second effect. With the coverage target disabled +... the coverage drought limit does not kick in, so it is also the time target that +... is responsible for stopping the trace generation without getting stuck in an +... infinite loop. +Suite Setup Treat this test suite Model-based coverage_target=0 time_target=0.1 sec +Suite Teardown Should be equal ${scenario_count} ${1} +Test Teardown Set suite variable ${scenario_count} ${scenario_count+1} +Resource ../../../resources/birthday_cards_flat.resource +Library robotmbt + +*** variables *** +${scenario_count} ${0} + +*** Test Cases *** +Buying a card + When someone buys a birthday card + then there is a blank birthday card available + +Someone writes their name on the card + Given there is a birthday card + when Someone writes their name on the birthday card + then the birthday card has 'Someone' written on it + +At least 3 people can write their name on the card + Given the birthday card has 2 names written on it + when someone writes their name on the birthday card + then the birthday card has 3 names written on it diff --git a/robotmbt/suiteprocessors.py b/robotmbt/suiteprocessors.py index f70db68..37731dc 100644 --- a/robotmbt/suiteprocessors.py +++ b/robotmbt/suiteprocessors.py @@ -32,9 +32,11 @@ import copy import random +import time from robot.api import logger from robot.errors import TimeoutExceeded +from robot.utils import timestr_to_secs from . import modeller from .modelspace import ModelSpace @@ -48,13 +50,20 @@ class SuiteProcessor: + def __init__(self): + self.scenario_count: int = 0 + self.commit_count: int = 0 + self.coverage_target: int = 1 + self.scenario_target: int = 0 + self.time_target: float = 0 + def process_test_suite(self, in_suite: Suite, **kwargs) -> Suite: self._handle_target_options(**kwargs) self.scenario_count = in_suite.scenario_count() # Counts the scenarios committed by the runner. I.e. the scenarios that are scheduled for execution # and cannot be touched anymore - self.commit_count: int = 0 + self.commit_count = 0 return Suite('not implemented') def next_scenario_request(self): @@ -94,17 +103,21 @@ def are_all_targets_reached(self, committed_only: bool = True) -> bool: return False if self.scenario_target and self.commit_count < self.scenario_target: return False + if self.time_target and time.time() < self.time_target: + return False return True def _handle_target_options(self, coverage_target: str | int | None = 1, - scenario_target: str | int | None = None, + scenario_target: str | int = 0, + time_target: str | None = None, **kwargs): self.coverage_target = 0 if coverage_target is None else int(coverage_target) if self.coverage_target not in [0, 1]: logger.warn(f"Unsupported coverage target request '{coverage_target}'. Using default coverage target of 1") self.coverage_target = 1 - self.scenario_target = 0 if scenario_target is None else int(scenario_target) + self.scenario_target = int(scenario_target) + self.time_target = (time.time() + timestr_to_secs(time_target)) if time_target else 0 class Echo(SuiteProcessor): @@ -216,6 +229,8 @@ def scenarios_pending(self) -> int: def are_all_targets_reached(self, tracestate: TraceState | None = None, committed_only: bool = True) -> bool: if tracestate is None: tracestate = self.tracestate + if self.time_target and time.time() < self.time_target: + return False if committed_only: if self.coverage_target and not tracestate[self.commit_count-1].coverage_reached: return False From f9f84e0f85e8222b2d1b25f32e7e432bdb959b55 Mon Sep 17 00:00:00 2001 From: JFoederer <32476108+JFoederer@users.noreply.github.com> Date: Fri, 19 Jun 2026 18:15:28 +0200 Subject: [PATCH 19/19] bump version to 0.14 --- pyproject.toml | 2 +- robotmbt/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d742728..f8fd9fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ build-backend = "setuptools.build_meta" [project] name = "robotframework-mbt" -version = "0.13.0" +version = "0.14.0" description = "Model-Based Testing in Robot framework with test case generation" readme = "README.md" authors = [{ name = "Johan Foederer", email = "github@famfoe.nl" }] diff --git a/robotmbt/version.py b/robotmbt/version.py index 0987cba..aab78c2 100644 --- a/robotmbt/version.py +++ b/robotmbt/version.py @@ -1 +1 @@ -VERSION: str = '0.13.0' +VERSION: str = '0.14.0'