diff --git a/.gitignore b/.gitignore index 727c718a..adfc9dbd 100644 --- a/.gitignore +++ b/.gitignore @@ -147,9 +147,6 @@ dmypy.json # Cython debug symbols cython_debug/ -# PyCharm -# JetBrains specific template is maintainted in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +# IDE ignores +.VScode/ +.idea/ diff --git a/atest/robotMBT tests/01__echo_suite/01__Single test/Echo_single_test_case.robot b/atest/robotMBT tests/01__echo_suite/01__Single_test/Echo_single_test_case.robot similarity index 100% rename from atest/robotMBT tests/01__echo_suite/01__Single test/Echo_single_test_case.robot rename to atest/robotMBT tests/01__echo_suite/01__Single_test/Echo_single_test_case.robot diff --git a/atest/robotMBT tests/01__echo_suite/01__Single test/__init__.robot b/atest/robotMBT tests/01__echo_suite/01__Single_test/__init__.robot similarity index 100% rename from atest/robotMBT tests/01__echo_suite/01__Single test/__init__.robot rename to atest/robotMBT tests/01__echo_suite/01__Single_test/__init__.robot diff --git a/atest/robotMBT tests/01__echo_suite/01__Single test/verify_result.robot b/atest/robotMBT tests/01__echo_suite/01__Single_test/verify_result.robot similarity index 100% rename from atest/robotMBT tests/01__echo_suite/01__Single test/verify_result.robot rename to atest/robotMBT tests/01__echo_suite/01__Single_test/verify_result.robot diff --git a/atest/robotMBT tests/01__echo_suite/05__Multiple_folders with setups/Echo_multiple_folder_suites/01__First_folder_suite/01__Echo_single_test_case.robot b/atest/robotMBT tests/01__echo_suite/05__Multiple_folders_with_setups/Echo_multiple_folder_suites/01__First_folder_suite/01__Echo_single_test_case.robot similarity index 100% rename from atest/robotMBT tests/01__echo_suite/05__Multiple_folders with setups/Echo_multiple_folder_suites/01__First_folder_suite/01__Echo_single_test_case.robot rename to atest/robotMBT tests/01__echo_suite/05__Multiple_folders_with_setups/Echo_multiple_folder_suites/01__First_folder_suite/01__Echo_single_test_case.robot diff --git a/atest/robotMBT tests/01__echo_suite/05__Multiple_folders with setups/Echo_multiple_folder_suites/01__First_folder_suite/02__Echo_multiple_test_cases.robot b/atest/robotMBT tests/01__echo_suite/05__Multiple_folders_with_setups/Echo_multiple_folder_suites/01__First_folder_suite/02__Echo_multiple_test_cases.robot similarity index 100% rename from atest/robotMBT tests/01__echo_suite/05__Multiple_folders with setups/Echo_multiple_folder_suites/01__First_folder_suite/02__Echo_multiple_test_cases.robot rename to atest/robotMBT tests/01__echo_suite/05__Multiple_folders_with_setups/Echo_multiple_folder_suites/01__First_folder_suite/02__Echo_multiple_test_cases.robot diff --git a/atest/robotMBT tests/01__echo_suite/05__Multiple_folders with setups/Echo_multiple_folder_suites/01__First_folder_suite/__init__.robot b/atest/robotMBT tests/01__echo_suite/05__Multiple_folders_with_setups/Echo_multiple_folder_suites/01__First_folder_suite/__init__.robot similarity index 100% rename from atest/robotMBT tests/01__echo_suite/05__Multiple_folders with setups/Echo_multiple_folder_suites/01__First_folder_suite/__init__.robot rename to atest/robotMBT tests/01__echo_suite/05__Multiple_folders_with_setups/Echo_multiple_folder_suites/01__First_folder_suite/__init__.robot diff --git a/atest/robotMBT tests/01__echo_suite/05__Multiple_folders with setups/Echo_multiple_folder_suites/02__Second_folder_suite/01__Echo_single_test_case.robot b/atest/robotMBT tests/01__echo_suite/05__Multiple_folders_with_setups/Echo_multiple_folder_suites/02__Second_folder_suite/01__Echo_single_test_case.robot similarity index 100% rename from atest/robotMBT tests/01__echo_suite/05__Multiple_folders with setups/Echo_multiple_folder_suites/02__Second_folder_suite/01__Echo_single_test_case.robot rename to atest/robotMBT tests/01__echo_suite/05__Multiple_folders_with_setups/Echo_multiple_folder_suites/02__Second_folder_suite/01__Echo_single_test_case.robot diff --git a/atest/robotMBT tests/01__echo_suite/05__Multiple_folders with setups/Echo_multiple_folder_suites/02__Second_folder_suite/02__Echo_multiple_test_cases.robot b/atest/robotMBT tests/01__echo_suite/05__Multiple_folders_with_setups/Echo_multiple_folder_suites/02__Second_folder_suite/02__Echo_multiple_test_cases.robot similarity index 100% rename from atest/robotMBT tests/01__echo_suite/05__Multiple_folders with setups/Echo_multiple_folder_suites/02__Second_folder_suite/02__Echo_multiple_test_cases.robot rename to atest/robotMBT tests/01__echo_suite/05__Multiple_folders_with_setups/Echo_multiple_folder_suites/02__Second_folder_suite/02__Echo_multiple_test_cases.robot diff --git a/atest/robotMBT tests/01__echo_suite/05__Multiple_folders with setups/Echo_multiple_folder_suites/03__Echo_single_test_case.robot b/atest/robotMBT tests/01__echo_suite/05__Multiple_folders_with_setups/Echo_multiple_folder_suites/03__Echo_single_test_case.robot similarity index 100% rename from atest/robotMBT tests/01__echo_suite/05__Multiple_folders with setups/Echo_multiple_folder_suites/03__Echo_single_test_case.robot rename to atest/robotMBT tests/01__echo_suite/05__Multiple_folders_with_setups/Echo_multiple_folder_suites/03__Echo_single_test_case.robot diff --git a/atest/robotMBT tests/01__echo_suite/05__Multiple_folders with setups/Echo_multiple_folder_suites/04__Echo_multiple_test_cases.robot b/atest/robotMBT tests/01__echo_suite/05__Multiple_folders_with_setups/Echo_multiple_folder_suites/04__Echo_multiple_test_cases.robot similarity index 100% rename from atest/robotMBT tests/01__echo_suite/05__Multiple_folders with setups/Echo_multiple_folder_suites/04__Echo_multiple_test_cases.robot rename to atest/robotMBT tests/01__echo_suite/05__Multiple_folders_with_setups/Echo_multiple_folder_suites/04__Echo_multiple_test_cases.robot diff --git a/atest/robotMBT tests/01__echo_suite/05__Multiple_folders with setups/Echo_multiple_folder_suites/__init__.robot b/atest/robotMBT tests/01__echo_suite/05__Multiple_folders_with_setups/Echo_multiple_folder_suites/__init__.robot similarity index 100% rename from atest/robotMBT tests/01__echo_suite/05__Multiple_folders with setups/Echo_multiple_folder_suites/__init__.robot rename to atest/robotMBT tests/01__echo_suite/05__Multiple_folders_with_setups/Echo_multiple_folder_suites/__init__.robot diff --git a/atest/robotMBT tests/01__echo_suite/05__Multiple_folders with setups/Verify_result.robot b/atest/robotMBT tests/01__echo_suite/05__Multiple_folders_with_setups/Verify_result.robot similarity index 100% rename from atest/robotMBT tests/01__echo_suite/05__Multiple_folders with setups/Verify_result.robot rename to atest/robotMBT tests/01__echo_suite/05__Multiple_folders_with_setups/Verify_result.robot diff --git a/atest/robotMBT tests/01__echo_suite/05__Multiple_folders with setups/__init__.robot b/atest/robotMBT tests/01__echo_suite/05__Multiple_folders_with_setups/__init__.robot similarity index 100% rename from atest/robotMBT tests/01__echo_suite/05__Multiple_folders with setups/__init__.robot rename to atest/robotMBT tests/01__echo_suite/05__Multiple_folders_with_setups/__init__.robot diff --git a/atest/robotMBT tests/01__echo_suite/06__Multiple_folders with teardown/Echo_multiple_folder_suites/01__First_folder_suite/01__Echo_single_test_case.robot b/atest/robotMBT tests/01__echo_suite/06__Multiple_folders_with_teardown/Echo_multiple_folder_suites/01__First_folder_suite/01__Echo_single_test_case.robot similarity index 100% rename from atest/robotMBT tests/01__echo_suite/06__Multiple_folders with teardown/Echo_multiple_folder_suites/01__First_folder_suite/01__Echo_single_test_case.robot rename to atest/robotMBT tests/01__echo_suite/06__Multiple_folders_with_teardown/Echo_multiple_folder_suites/01__First_folder_suite/01__Echo_single_test_case.robot diff --git a/atest/robotMBT tests/01__echo_suite/06__Multiple_folders with teardown/Echo_multiple_folder_suites/01__First_folder_suite/02__Echo_multiple_test_cases.robot b/atest/robotMBT tests/01__echo_suite/06__Multiple_folders_with_teardown/Echo_multiple_folder_suites/01__First_folder_suite/02__Echo_multiple_test_cases.robot similarity index 100% rename from atest/robotMBT tests/01__echo_suite/06__Multiple_folders with teardown/Echo_multiple_folder_suites/01__First_folder_suite/02__Echo_multiple_test_cases.robot rename to atest/robotMBT tests/01__echo_suite/06__Multiple_folders_with_teardown/Echo_multiple_folder_suites/01__First_folder_suite/02__Echo_multiple_test_cases.robot diff --git a/atest/robotMBT tests/01__echo_suite/06__Multiple_folders with teardown/Echo_multiple_folder_suites/01__First_folder_suite/__init__.robot b/atest/robotMBT tests/01__echo_suite/06__Multiple_folders_with_teardown/Echo_multiple_folder_suites/01__First_folder_suite/__init__.robot similarity index 100% rename from atest/robotMBT tests/01__echo_suite/06__Multiple_folders with teardown/Echo_multiple_folder_suites/01__First_folder_suite/__init__.robot rename to atest/robotMBT tests/01__echo_suite/06__Multiple_folders_with_teardown/Echo_multiple_folder_suites/01__First_folder_suite/__init__.robot diff --git a/atest/robotMBT tests/01__echo_suite/06__Multiple_folders with teardown/Echo_multiple_folder_suites/02__Second_folder_suite/01__Echo_single_test_case.robot b/atest/robotMBT tests/01__echo_suite/06__Multiple_folders_with_teardown/Echo_multiple_folder_suites/02__Second_folder_suite/01__Echo_single_test_case.robot similarity index 100% rename from atest/robotMBT tests/01__echo_suite/06__Multiple_folders with teardown/Echo_multiple_folder_suites/02__Second_folder_suite/01__Echo_single_test_case.robot rename to atest/robotMBT tests/01__echo_suite/06__Multiple_folders_with_teardown/Echo_multiple_folder_suites/02__Second_folder_suite/01__Echo_single_test_case.robot diff --git a/atest/robotMBT tests/01__echo_suite/06__Multiple_folders with teardown/Echo_multiple_folder_suites/02__Second_folder_suite/02__Echo_multiple_test_cases.robot b/atest/robotMBT tests/01__echo_suite/06__Multiple_folders_with_teardown/Echo_multiple_folder_suites/02__Second_folder_suite/02__Echo_multiple_test_cases.robot similarity index 100% rename from atest/robotMBT tests/01__echo_suite/06__Multiple_folders with teardown/Echo_multiple_folder_suites/02__Second_folder_suite/02__Echo_multiple_test_cases.robot rename to atest/robotMBT tests/01__echo_suite/06__Multiple_folders_with_teardown/Echo_multiple_folder_suites/02__Second_folder_suite/02__Echo_multiple_test_cases.robot diff --git a/atest/robotMBT tests/01__echo_suite/06__Multiple_folders with teardown/Echo_multiple_folder_suites/02__Second_folder_suite/__init__.robot b/atest/robotMBT tests/01__echo_suite/06__Multiple_folders_with_teardown/Echo_multiple_folder_suites/02__Second_folder_suite/__init__.robot similarity index 100% rename from atest/robotMBT tests/01__echo_suite/06__Multiple_folders with teardown/Echo_multiple_folder_suites/02__Second_folder_suite/__init__.robot rename to atest/robotMBT tests/01__echo_suite/06__Multiple_folders_with_teardown/Echo_multiple_folder_suites/02__Second_folder_suite/__init__.robot diff --git a/atest/robotMBT tests/01__echo_suite/06__Multiple_folders with teardown/Echo_multiple_folder_suites/03__Echo_single_test_case.robot b/atest/robotMBT tests/01__echo_suite/06__Multiple_folders_with_teardown/Echo_multiple_folder_suites/03__Echo_single_test_case.robot similarity index 100% rename from atest/robotMBT tests/01__echo_suite/06__Multiple_folders with teardown/Echo_multiple_folder_suites/03__Echo_single_test_case.robot rename to atest/robotMBT tests/01__echo_suite/06__Multiple_folders_with_teardown/Echo_multiple_folder_suites/03__Echo_single_test_case.robot diff --git a/atest/robotMBT tests/01__echo_suite/06__Multiple_folders with teardown/Echo_multiple_folder_suites/04__Echo_multiple_test_cases.robot b/atest/robotMBT tests/01__echo_suite/06__Multiple_folders_with_teardown/Echo_multiple_folder_suites/04__Echo_multiple_test_cases.robot similarity index 100% rename from atest/robotMBT tests/01__echo_suite/06__Multiple_folders with teardown/Echo_multiple_folder_suites/04__Echo_multiple_test_cases.robot rename to atest/robotMBT tests/01__echo_suite/06__Multiple_folders_with_teardown/Echo_multiple_folder_suites/04__Echo_multiple_test_cases.robot diff --git a/atest/robotMBT tests/01__echo_suite/06__Multiple_folders with teardown/Echo_multiple_folder_suites/__init__.robot b/atest/robotMBT tests/01__echo_suite/06__Multiple_folders_with_teardown/Echo_multiple_folder_suites/__init__.robot similarity index 100% rename from atest/robotMBT tests/01__echo_suite/06__Multiple_folders with teardown/Echo_multiple_folder_suites/__init__.robot rename to atest/robotMBT tests/01__echo_suite/06__Multiple_folders_with_teardown/Echo_multiple_folder_suites/__init__.robot diff --git a/atest/robotMBT tests/01__echo_suite/06__Multiple_folders with teardown/Verify_result.robot b/atest/robotMBT tests/01__echo_suite/06__Multiple_folders_with_teardown/Verify_result.robot similarity index 100% rename from atest/robotMBT tests/01__echo_suite/06__Multiple_folders with teardown/Verify_result.robot rename to atest/robotMBT tests/01__echo_suite/06__Multiple_folders_with_teardown/Verify_result.robot diff --git a/atest/robotMBT tests/01__echo_suite/06__Multiple_folders with teardown/__init__.robot b/atest/robotMBT tests/01__echo_suite/06__Multiple_folders_with_teardown/__init__.robot similarity index 100% rename from atest/robotMBT tests/01__echo_suite/06__Multiple_folders with teardown/__init__.robot rename to atest/robotMBT tests/01__echo_suite/06__Multiple_folders_with_teardown/__init__.robot diff --git a/atest/robotMBT tests/07__processor_options/04__batch_size/trace_extension_is_tagged.robot b/atest/robotMBT tests/07__processor_options/04__batch_size/01__trace_extension_is_tagged.robot similarity index 100% rename from atest/robotMBT tests/07__processor_options/04__batch_size/trace_extension_is_tagged.robot rename to atest/robotMBT tests/07__processor_options/04__batch_size/01__trace_extension_is_tagged.robot diff --git a/atest/robotMBT tests/07__processor_options/04__batch_size/02__fail_on_unreached_target.robot b/atest/robotMBT tests/07__processor_options/04__batch_size/02__fail_on_unreached_target.robot new file mode 100644 index 00000000..144825e7 --- /dev/null +++ b/atest/robotMBT tests/07__processor_options/04__batch_size/02__fail_on_unreached_target.robot @@ -0,0 +1,45 @@ +*** Settings *** +Documentation With the introduction of on-the-fly trace generation, a test run can be started +... before it is known whether the requested coverage target can be reached. If the +... test run finishes before the coverage target is achieved, a special scenario is +... inserted that will fail. This ensures a failed test run, even if all tests up to +... this point passed. This test suite checks that mechanism. +Suite Setup Treat this test suite Model-based batch_size=2 coverage_target=1 +Suite Teardown Should Be True ${confirmed_pass} # set by listener after catching intended failure +Library robotmbt +Library catch_intentional_fail.py + + +*** Variables *** +${confirmed_pass} ${False} + + +*** Test Cases *** +Scenario 1 + First + +Scenario 2 + Second + +Impossible Scenario + Block here + + +*** Keywords *** +First + [Documentation] *model info* + ... :IN: None + ... :OUT: new item + No Operation + +Second + [Documentation] *model info* + ... :IN: item + ... :OUT: None + No Operation + +Block here + [Documentation] *model info* + ... :IN: False + ... :OUT: None + No Operation diff --git a/atest/robotMBT tests/07__processor_options/04__batch_size/catch_intentional_fail.py b/atest/robotMBT tests/07__processor_options/04__batch_size/catch_intentional_fail.py new file mode 100644 index 00000000..12ccfbef --- /dev/null +++ b/atest/robotMBT tests/07__processor_options/04__batch_size/catch_intentional_fail.py @@ -0,0 +1,18 @@ +from robot.api import logger +from robot.api.deco import library +from robot.libraries.BuiltIn import BuiltIn + + +@library(scope='SUITE', listener='SELF') +class ForcedFailListener: + ROBOT_LISTENER_PRIORITY = 2 # Set elevated priority, so other listeners (e.g. RIDE) also see the pass + + def end_test(self, tc, result): + if tc.name == "Confirm exit criteria": + if result.status == 'FAIL' and result.message == "Not all targets achieved": + BuiltIn().set_suite_variable('${confirmed_pass}', True) + result.status = 'PASS' + logger.info("Expected failure confirmed. Result flipped to PASS.") + else: + result.status = 'FAIL' + result.message = "Expected failure remained unconfirmed" diff --git a/atest/robotMBT tests/07__processor_options/04__batch_size/tag_listener.py b/atest/robotMBT tests/07__processor_options/04__batch_size/tag_listener.py index 373b3f99..874ce01d 100644 --- a/atest/robotMBT tests/07__processor_options/04__batch_size/tag_listener.py +++ b/atest/robotMBT tests/07__processor_options/04__batch_size/tag_listener.py @@ -25,5 +25,9 @@ def end_test(self, tc, result): if 'my tag' not in result.tags: result.status = 'FAIL' result.message = "Test tag 'my tag' missing" - BuiltIn().set_suite_variable('${confirmed_passes}', BuiltIn().get_variable_value('${confirmed_passes}') + 1) - logger.info("PASS confirmed by listener") + + if result.status == 'PASS': + BuiltIn().set_suite_variable('${confirmed_passes}', BuiltIn().get_variable_value('${confirmed_passes}') + 1) + logger.info("PASS confirmed by listener") + else: + logger.info("PASS remained unconfirmed by listener") diff --git a/pyproject.toml b/pyproject.toml index e8bdb6e9..14d60f8c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [build-system] requires = [ - "setuptools>=61.0.0", + "setuptools>=83.0.0", "wheel" ] build-backend = "setuptools.build_meta" @@ -11,9 +11,9 @@ version = "1.0.0" description = "Model-Based Testing in Robot Framework with test case generation" readme = "README.md" authors = [{ name = "Johan Foederer", email = "github@famfoe.nl" }] -license = { file = "LICENSE" } +license = "BSD-3-Clause" +license-files = ["LICENSE"] classifiers = [ - "License :: OSI Approved :: BSD License", "Programming Language :: Python :: 3", "Operating System :: OS Independent", ] diff --git a/robotmbt/suiteprocessors.py b/robotmbt/suiteprocessors.py index 37731dc1..e82b1588 100644 --- a/robotmbt/suiteprocessors.py +++ b/robotmbt/suiteprocessors.py @@ -36,7 +36,7 @@ from robot.api import logger from robot.errors import TimeoutExceeded -from robot.utils import timestr_to_secs +from robot.utils import timestr_to_secs, secs_to_timestr from . import modeller from .modelspace import ModelSpace @@ -51,27 +51,40 @@ class SuiteProcessor: def __init__(self): + self.start_time: float = 0 + self.target_duration: float = 0 self.scenario_count: int = 0 + # commit_count counts the scenarios committed by the runner. I.e. the scenarios that are + # scheduled for execution and cannot be touched anymore. self.commit_count: int = 0 - self.coverage_target: int = 1 - self.scenario_target: int = 0 - self.time_target: float = 0 + self.end_conditions: dict[str, int | float] = { + "coverage_target": 1, + "scenario_target": 0, + "time_target": 0.0 + } def process_test_suite(self, in_suite: Suite, **kwargs) -> Suite: self._handle_target_options(**kwargs) self.scenario_count = in_suite.scenario_count() - # Counts the scenarios committed by the runner. I.e. the scenarios that are scheduled for execution - # and cannot be touched anymore self.commit_count = 0 + self.scenarios_requested = False return Suite('not implemented') - def next_scenario_request(self): + def next_scenario_request(self) -> int: """Indicates the wish for (at least) one more scenario and triggers trace genaration when needed.""" # This basic implementation assumes that the complete target test suite is returned directly # by process_test_suite() in an overridden method. No further generation is triggered. - if self.scenario_count >= self.commit_count + 1: + if self.scenarios_requested: + return 0 + self.scenarios_requested = True + return self.scenario_count + + def commit_next_scenario(self) -> bool: + if self.scenario_count > self.commit_count: self.commit_count += 1 + return True + return False @property def scenarios_committed(self) -> int: @@ -97,27 +110,56 @@ def scenarios_pending(self) -> int: return self.scenario_count - self.commit_count def are_all_targets_reached(self, committed_only: bool = True) -> bool: - if not committed_only: - return True - if self.coverage_target and self.commit_count < self.scenario_count: - return False - if self.scenario_target and self.commit_count < self.scenario_target: - return False - if self.time_target and time.time() < self.time_target: - return False + for condition in self.end_conditions: + is_hit, _ = self.check_end_condition(condition, committed_only=committed_only) + if not is_hit: + return False return True + def target_summary(self): + status_chart = ["Progress towards run targets (actual/target):"] + for condition, threshold in self.end_conditions.items(): + if threshold: + is_hit, actual = self.check_end_condition(condition) + if condition == "time_target": + threshold = secs_to_timestr(self.target_duration, compact=True) + actual = secs_to_timestr(int(actual - self.start_time), compact=True) + status_chart.append(f"{condition}{'' if is_hit else ' not'} hit ({actual}/{threshold})") + return "\n ".join(status_chart) + + def progress_report(self): + logger.info(self.target_summary()) + + def check_end_condition(self, condition, committed_only: bool = True) -> tuple[bool, int | float]: + threshold = self.end_conditions[condition] + if not threshold: + return True, 0 + match condition: + case "coverage_target": + actual = self.commit_count // self.scenario_count if committed_only else 1 + case "scenario_target": + actual = self.commit_count if committed_only else self.scenario_count + case "time_target": + actual = time.time() + case _: + actual = 0 + + is_hit = actual >= threshold if condition.endswith("_target") else False + return is_hit, actual + def _handle_target_options(self, coverage_target: str | int | None = 1, scenario_target: str | int = 0, time_target: str | None = None, **kwargs): - self.coverage_target = 0 if coverage_target is None else int(coverage_target) - if self.coverage_target not in [0, 1]: + self.end_conditions["coverage_target"] = 0 if coverage_target is None else int(coverage_target) + if self.end_conditions["coverage_target"] not in [0, 1]: logger.warn(f"Unsupported coverage target request '{coverage_target}'. Using default coverage target of 1") - self.coverage_target = 1 - self.scenario_target = int(scenario_target) - self.time_target = (time.time() + timestr_to_secs(time_target)) if time_target else 0 + self.end_conditions["coverage_target"] = 1 + self.end_conditions["scenario_target"] = int(scenario_target) + self.start_time = time.time() + self.target_duration = timestr_to_secs(time_target) if time_target else 0 + self.end_conditions["time_target"] = (self.start_time + self.target_duration) if time_target else 0 class Echo(SuiteProcessor): @@ -169,6 +211,8 @@ def process_test_suite(self, in_suite: Suite, *, seed: str | int | bytes | bytea super().process_test_suite(in_suite, **kwargs) self.batch_size = int(batch_size) self._init_randomiser(seed) + self._graph_style = graph + self._export_graph = export_graph_data self._visualiser = self._init_visualiser(in_suite.name) if graph or export_graph_data else None self.out_suite = Suite(in_suite.name) @@ -198,7 +242,7 @@ def process_test_suite(self, in_suite: Suite, *, seed: str | int | bytes | bytea else: self.tracestate = TraceState([s.src_id for s in self.scenarios]) self.tracestate.unreached = direct_tracestate.unreached - logger.debug("Direct trace not discovered. Now exploring with loops, allowing repetition of scenarios.") + logger.debug("Discovery phase finished. Now exploring with loops, allowing repetition of scenarios.") self._generate_next_batch(self.batch_size) finally: # Draw the graph even when a timeout or user interrupt occurs if graph: @@ -210,13 +254,19 @@ def process_test_suite(self, in_suite: Suite, *, seed: str | int | bytes | bytea self._report_tracestate_wrapup() return self.out_suite - def next_scenario_request(self): - if len(self.tracestate) <= self.out_suite.scenario_count(): + def next_scenario_request(self) -> int: + pending_at_entry = self.scenarios_pending + if not pending_at_entry: self._generate_next_batch(self.batch_size) + return self.scenarios_pending - pending_at_entry + + def commit_next_scenario(self) -> bool: if len(self.tracestate) > self.out_suite.scenario_count(): self.out_suite.scenarios.append(self.tracestate[self.out_suite.scenario_count()].scenario) self.commit_count += 1 self.tracestate.rewind_limit += 1 + return True + return False @property def scenarios_committed(self) -> int: @@ -226,23 +276,48 @@ def scenarios_committed(self) -> int: def scenarios_pending(self) -> int: return len(self.tracestate) - self.out_suite.scenario_count() - def are_all_targets_reached(self, tracestate: TraceState | None = None, committed_only: bool = True) -> bool: - if tracestate is None: - tracestate = self.tracestate - if self.time_target and time.time() < self.time_target: - return False - if committed_only: - if self.coverage_target and not tracestate[self.commit_count-1].coverage_reached: - return False - if self.scenario_target and self.commit_count < self.scenario_target: - return False - else: - if self.coverage_target and not tracestate.coverage_reached(): - return False - if self.scenario_target and len(tracestate) < self.scenario_target: + def progress_report(self): + super().progress_report() + if self._graph_style: + self._write_visualisation(self._graph_style) + if self._export_graph: + self._export_graph_data(self._export_graph) + + def are_all_targets_reached(self, committed_only: bool = True) -> bool: + return self.all_targets_check_w_trace(self.tracestate, committed_only) + + def check_end_condition(self, condition: str, committed_only: bool = True) -> tuple[bool, int | float]: + return self.check_end_condition_w_trace(self.tracestate, condition, committed_only) + + def all_targets_check_w_trace(self, tracestate: TraceState, committed_only: bool = True) -> bool: + for condition in self.end_conditions: + is_hit, _ = self.check_end_condition_w_trace(tracestate, condition, committed_only=committed_only) + if not is_hit: return False return True + def check_end_condition_w_trace(self, tracestate: TraceState, condition: str, committed_only: bool = True) -> tuple[bool, int | float]: + match condition: + case "time_target": + return super().check_end_condition(condition, committed_only) + case "coverage_target": + if committed_only: + actual = 1 if tracestate[self.commit_count-1].coverage_reached else 0 + else: + actual = 1 if tracestate.coverage_reached() else 0 + case "scenario_target": + actual = self.commit_count if committed_only else len(tracestate) + case _: + actual = 0 + + threshold = self.end_conditions[condition] + if condition.endswith("_target"): + is_hit = actual >= threshold if threshold else True + else: + is_hit = False + + return is_hit, actual + def draw_graph_from_export_file(self, file_path: str, graph_style: str): self._visualiser = self._init_visualiser() if self._visualiser: @@ -326,7 +401,7 @@ def _search_direct_trace(self) -> TraceState: return longest def _discovery_ready(self, tracestate): - return self.are_all_targets_reached(tracestate, committed_only=False) or len(tracestate) >= self.batch_size + return self.all_targets_check_w_trace(tracestate, committed_only=False) or len(tracestate) >= self.batch_size def _longest_trace(self, tracestate_list: list[TraceState]) -> int: """returns the index of the trace that covers the most scenarios""" @@ -398,6 +473,7 @@ def _generate_next_batch(self, batchsize): tracestate = self.tracestate old_len = len(tracestate) self._update_visualisation(tracestate) + self._report_tracestate_to_user(tracestate) while len(tracestate) < old_len + batchsize and not self.are_all_targets_reached(committed_only=False): candidate_id = tracestate.next_candidate(retry=True, randomise=True) if candidate_id is None: @@ -405,7 +481,7 @@ def _generate_next_batch(self, batchsize): if not tracestate.can_rewind(): break tail = modeller.rewind(tracestate) - logger.debug(f"Having to roll back up to {tail.scenario.name if tail else 'the beginning'}") + logger.debug(f"Having to roll back up to: {tail.scenario.name if tail else 'the beginning'}") self._report_tracestate_to_user(tracestate) if tracestate.model: logger.debug(f"last state:\n{tracestate.model.get_status_text()}") @@ -425,7 +501,7 @@ def _generate_next_batch(self, batchsize): if self.__last_candidate_changed_nothing(tracestate): logger.debug("Repeated scenario did not change the model's state. Stop trying.") modeller.rewind(tracestate) - elif self.coverage_target and not self.tracestate.coverage_reached() and tracestate.coverage_drought > self.DROUGHT_LIMIT: + elif self.end_conditions["coverage_target"] and not self.tracestate.coverage_reached() and tracestate.coverage_drought > self.DROUGHT_LIMIT: logger.debug(f"Went too long without new coverage (>{self.DROUGHT_LIMIT}x). " "Roll back to last coverage increase and try something else.") modeller.rewind(tracestate, drought_recovery=True) diff --git a/robotmbt/suitereplacer.py b/robotmbt/suitereplacer.py index 843696d9..dde75fb2 100644 --- a/robotmbt/suitereplacer.py +++ b/robotmbt/suitereplacer.py @@ -94,6 +94,7 @@ def treat_model_based(self, **kwargs): self.__clearTestSuite(self.current_suite) self.mbt_anchor_suite = self.current_suite self.processor.next_scenario_request() + self.processor.commit_next_scenario() self.add_next_new(self.mbt_anchor_suite) @keyword("Set model-based options") @@ -237,30 +238,35 @@ def _end_test(self, test_case: rmodel.TestCase, result: robot.result.model.TestC if not isinstance(self.processor, SuiteProcessor): raise TypeError("processor must be of type SuiteProcessor") if self.processor.are_all_targets_reached(): + self.processor.progress_report() logger.info(f"{self.processor.scenarios_committed} Scenarios completed for model. All targets achieved.") return - committed_old = self.processor.scenarios_committed - pending_old = self.processor.scenarios_pending - if not pending_old: - logger.info(f"{committed_old} Scenario{'s' if committed_old != 1 else ''} completed. Looking to extend trace.") - self.processor.next_scenario_request() - committed = self.processor.scenarios_committed - pending = self.processor.scenarios_pending - new_total = committed + pending - old_total = committed_old + pending_old - if not pending_old and new_total == old_total: - logger.info(f"Trace could not be extended.") + scenarios_in_buffer = self.processor.scenarios_pending + if not scenarios_in_buffer: + committed = self.processor.scenarios_committed + logger.info(f"{committed} Scenario{'s' if committed != 1 else ''} completed. Looking to extend trace.") + self.processor.progress_report() + newly_added = self.processor.next_scenario_request() + if not scenarios_in_buffer and not newly_added: if not self.processor.are_all_targets_reached(): + logger.info(f"Trace could not be extended.") new_tc = self.current_suite.tests.create(name='Confirm exit criteria') + new_tc.body.create_keyword(name='Log', args=(self.processor.target_summary(),)) new_tc.body.create_keyword(name='Fail', args=('Not all targets achieved',)) self.mbt_anchor_suite = None return - if new_total > old_total: + if newly_added: result.tags.add('mbt trace extension') - logger.info(f"MBT trace generation prepared {new_total-old_total} new scenarios.") + logger.info(f"MBT trace generation prepared {newly_added} new scenario{'s' if newly_added != 1 else ''}.") + if not scenarios_in_buffer and self.processor.are_all_targets_reached(): + logger.info("Processing stopped. All targets achieved.") + return + self.processor.commit_next_scenario() try: self.add_test(next(self.test_case_gen[-1]), self.current_suite) except StopIteration: + # Happens when the newly committed scenario is in the next test suite. + # The 'add test' for that suite is called in the end_suite listener. pass diff --git a/robotmbt/visualise/models.py b/robotmbt/visualise/models.py index f30c7cc9..fea08e09 100644 --- a/robotmbt/visualise/models.py +++ b/robotmbt/visualise/models.py @@ -259,7 +259,7 @@ def _sanity_check(self, scen: ScenarioInfo, state: StateInfo, after: str): logger.warn( f'TraceInfo got out of sync after {after}\nExpected state: {prev_state}\nActual state: {state}') - def export_graph(self, dir: str = '', atest: bool = False) -> str | None: + def export_graph(self, dir: str = '', atest: bool = False) -> str: encoded_instance = jsonpickle.encode(self) name = self.model_name.lower().replace(' ', '_') if atest: @@ -281,9 +281,10 @@ def export_graph(self, dir: str = '', atest: bool = False) -> str | None: if not os.path.exists(dir): os.makedirs(dir) - with open(f"{dir}{name}.json", "w") as f: + filename = f"{dir}{name}.json" + with open(filename, "w") as f: f.write(encoded_instance) - return None + return filename @staticmethod def import_graph_from_file(file_path: str): diff --git a/robotmbt/visualise/visualiser.py b/robotmbt/visualise/visualiser.py index da0f33a5..a85ed304 100644 --- a/robotmbt/visualise/visualiser.py +++ b/robotmbt/visualise/visualiser.py @@ -62,8 +62,8 @@ def load_from_file(self, file_path: str): """ self.trace_info = TraceInfo.import_graph_from_file(file_path) - def export_to_file(self, file_path: str): - self.trace_info.export_graph(file_path) + def export_to_file(self, file_path: str) -> str: + return self.trace_info.export_graph(file_path) def update_trace(self, trace: TraceState): """ diff --git a/utest/test_suiteprocessors.py b/utest/test_suiteprocessors.py index 77be7c92..961b5f1b 100644 --- a/utest/test_suiteprocessors.py +++ b/utest/test_suiteprocessors.py @@ -124,14 +124,18 @@ def test_batch_size_1(self): out_suite = self.processor.process_test_suite(self.suite, scenario_target=3, batch_size=1) self.assertEqual(self.processor.scenarios_pending, 1) self.processor.next_scenario_request() + self.processor.commit_next_scenario() + self.processor.commit_next_scenario() self.assertEqual(out_suite.scenario_count(), 1) self.assertEqual(self.processor.scenarios_committed, 1) self.assertEqual(self.processor.scenarios_pending, 0) self.processor.next_scenario_request() + self.processor.commit_next_scenario() self.assertEqual(out_suite.scenario_count(), 2) self.assertEqual(self.processor.scenarios_committed, 2) self.assertEqual(self.processor.scenarios_pending, 0) self.processor.next_scenario_request() + self.processor.commit_next_scenario() self.assertEqual(out_suite.scenario_count(), 3) self.assertEqual(self.processor.scenarios_committed, 3) self.assertEqual(self.processor.scenarios_pending, 0) @@ -142,48 +146,61 @@ def test_batch_size_2_last_batch_not_full(self): out_suite = self.processor.process_test_suite(self.suite, scenario_target=3, batch_size=2) self.assertEqual(self.processor.scenarios_pending, 2) self.processor.next_scenario_request() + self.processor.commit_next_scenario() self.assertEqual(out_suite.scenario_count(), 1) self.assertEqual(self.processor.scenarios_committed, 1) self.assertEqual(self.processor.scenarios_pending, 1) self.processor.next_scenario_request() + self.processor.commit_next_scenario() self.assertEqual(self.processor.scenarios_pending, 0) self.processor.next_scenario_request() + self.processor.commit_next_scenario() self.assertEqual(self.processor.scenarios_pending, 0) self.assertEqual(out_suite.scenario_count(), 3) def test_batch_size_2_last_batch_full(self): out_suite = self.processor.process_test_suite(self.suite, scenario_target=4, batch_size=2) self.processor.next_scenario_request() + self.processor.commit_next_scenario() self.assertEqual(out_suite.scenario_count(), 1) self.assertEqual(self.processor.scenarios_committed, 1) self.assertEqual(self.processor.scenarios_pending, 1) self.processor.next_scenario_request() + self.processor.commit_next_scenario() self.assertEqual(self.processor.scenarios_pending, 0) self.processor.next_scenario_request() + self.processor.commit_next_scenario() self.assertEqual(self.processor.scenarios_pending, 1) self.processor.next_scenario_request() + self.processor.commit_next_scenario() self.assertEqual(self.processor.scenarios_pending, 0) self.assertEqual(out_suite.scenario_count(), 4) def test_batch_size_10(self): out_suite = self.processor.process_test_suite(self.suite, scenario_target=15, batch_size=10) self.processor.next_scenario_request() + self.processor.commit_next_scenario() self.assertEqual(out_suite.scenario_count(), 1) self.assertEqual(self.processor.scenarios_pending, 9) for _ in range(9): self.processor.next_scenario_request() + self.processor.commit_next_scenario() self.assertEqual(out_suite.scenario_count(), 10) self.assertEqual(self.processor.scenarios_pending, 0) self.processor.next_scenario_request() + self.processor.commit_next_scenario() self.assertEqual(self.processor.scenarios_pending, 4) def test_batch_size_3_is_trace_length(self): out_suite = self.processor.process_test_suite(self.suite, scenario_target=3, batch_size=3) self.assertEqual(self.processor.scenarios_pending, 3) self.processor.next_scenario_request() + self.processor.commit_next_scenario() self.assertEqual(self.processor.scenarios_pending, 2) self.processor.next_scenario_request() + self.processor.commit_next_scenario() self.processor.next_scenario_request() + self.processor.commit_next_scenario() self.assertEqual(out_suite.scenario_count(), 3) self.assertListEqual([s.name[:self.scenario_name_without_rep_count] for s in out_suite.scenarios], ['init scenario'] + ['body scenario']*(out_suite.scenario_count()-1)) @@ -191,27 +208,36 @@ def test_batch_size_3_is_trace_length(self): def test_requesting_beyond_targets_has_no_effect(self): out_suite = self.processor.process_test_suite(self.suite, scenario_target=3, batch_size=3) self.processor.next_scenario_request() + self.processor.commit_next_scenario() self.processor.next_scenario_request() + self.processor.commit_next_scenario() self.assertFalse(self.processor.are_all_targets_reached()) self.processor.next_scenario_request() + committed = self.processor.commit_next_scenario() + self.assertTrue(committed) self.assertEqual(out_suite.scenario_count(), 3) self.assertTrue(self.processor.are_all_targets_reached()) self.processor.next_scenario_request() + self.assertFalse(self.processor.commit_next_scenario()) self.assertTrue(self.processor.are_all_targets_reached()) self.assertEqual(out_suite.scenario_count(), 3) def test_multi_batch(self): """Check some variations in batch size versus target size""" for target, batch in [(1, 1), # Smallest target and batch - (23, 3), # Multiple batches needed te completer + (24, 6), # Multiple batches needed to complete (exact fit) + (23, 3), # Multiple batches needed to complete (non-divisible) (10, 24), # Batch size exceeds target size (15, 14), # Batch size just not enough - (16, 16) # Batch size equals target size + (16, 16), # Batch size equals target size + (17, 1) # Repeated batch of one ]: out_suite = self.processor.process_test_suite(self.suite, coverage_target=0, scenario_target=target, batch_size=batch) while not self.processor.are_all_targets_reached(): - self.processor.next_scenario_request() + newly_added = self.processor.next_scenario_request() + self.assertLessEqual(newly_added, batch) + self.processor.commit_next_scenario() self.assertEqual(out_suite.scenario_count(), target) self.assertListEqual([s.name[:self.scenario_name_without_rep_count] for s in out_suite.scenarios], ['init scenario'] + ['body scenario']*(out_suite.scenario_count()-1))