From b0e61cbc28c95946a2dbcf2213afc24aafc225f8 Mon Sep 17 00:00:00 2001 From: Syed Fasiuddin Date: Wed, 31 Dec 2025 13:56:56 +0000 Subject: [PATCH] test: add random testing for `evaluator` The evaluator module internally uses all the other modules and hence if we generate tests for this one then all other modules are automatically tested. The features cannot be tested individually because of how the code is structured and its interdependence between all other user stories and all of the other code. The commands used to generate tests is: ``` PYNGUIN_DANGER_AWARE="1" PYTHONHASHSEED="0" pynguin \ --project-path . --module-name src.evaluator \ --output-path 259045253/test/blackbox/random_testing \ --algorithm RANDOM --assertion-generation NONE \ --seed 0 --maximum-search-time 60 -v ``` And the command used to generate coverage report is: ``` coverage run -m pytest \ 259045253/test/blackbox/random_testing/test_src_evaluator.py \ && coverage report \ -m > 259045253/test/blackbox/random_testing/coverage.txt ``` The change in `src/evaluator.py` file was made because I kept getting some python byte code error. --- .../test/blackbox/random_testing/.coverage | Bin 0 -> 53248 bytes .../test/blackbox/random_testing/coverage.txt | 13 + .../pynguin-report/cov_report.html | 317 + .../pynguin-report/cov_report.xml | 17 + .../pynguin-report/pynguin-cli-params.txt | 290 + .../pynguin-report/pynguin-config.toml | 192 + .../pynguin-report/pynguin-config.txt | 88 + .../evaluator/evaluator_result.json | 1 + .../signatures/lexer/lexer_result.json | 1 + .../signatures/src/src_result.json | 1 + .../pynguin-report/statistics.csv | 15 + .../random_testing/test_src_evaluator.py | 7946 +++++++++++++++++ src/evaluator.py | 5 +- 13 files changed, 8885 insertions(+), 1 deletion(-) create mode 100644 259045253/test/blackbox/random_testing/.coverage create mode 100644 259045253/test/blackbox/random_testing/coverage.txt create mode 100644 259045253/test/blackbox/random_testing/pynguin-report/cov_report.html create mode 100644 259045253/test/blackbox/random_testing/pynguin-report/cov_report.xml create mode 100644 259045253/test/blackbox/random_testing/pynguin-report/pynguin-cli-params.txt create mode 100644 259045253/test/blackbox/random_testing/pynguin-report/pynguin-config.toml create mode 100644 259045253/test/blackbox/random_testing/pynguin-report/pynguin-config.txt create mode 100644 259045253/test/blackbox/random_testing/pynguin-report/signatures/evaluator/evaluator_result.json create mode 100644 259045253/test/blackbox/random_testing/pynguin-report/signatures/lexer/lexer_result.json create mode 100644 259045253/test/blackbox/random_testing/pynguin-report/signatures/src/src_result.json create mode 100644 259045253/test/blackbox/random_testing/pynguin-report/statistics.csv create mode 100644 259045253/test/blackbox/random_testing/test_src_evaluator.py diff --git a/259045253/test/blackbox/random_testing/.coverage b/259045253/test/blackbox/random_testing/.coverage new file mode 100644 index 0000000000000000000000000000000000000000..4b452159748f2f0bb17d2e9aa6ebfc7c439e34a2 GIT binary patch literal 53248 zcmeI)%WoS+90%}SJGEoSW{axGilQoWfi!9y*9{da93apfDk_yi=>-XQydEb@_mS-E z=5avCEmB1h{{iBE;y>Wn6E_Z3d*JumhaYWR^4-I>{$&%C@|XTQF4%ZZfe z2cah;v2C0;Ow;&O2*WU{^sLh}U&^$ykiVh7=GgkQ)vB>|^mE<1XVgl+8rFC9+g7vo zTmAi|KWbj($em%8iv3^WhJX?7AunrLDsBwf^~6yE}Jx z#oe9HZtaS6o48aJv^1MyM})z#I8dSJJFXIr-*apkIsTrA_EjD_)~d%79c`njjstEb z@p-M|^e9%O_9%&i(D7t=DjulQ%_Kl}v`C#q=>|$fIeR{b5SOy(8zNMF6)N9WI?cr; zr?*kvc=Thn+_-STe5{f($A}X`;?F!ggwrEH`9gT(rJ=O$bAj%sF|%Wh>5L+u5i<*( zMebAP{2*r?MdU+M~)F!``A0+%ApwPfumqr%xQ6>AS>(}*UcGqIV$f&b>`$; zqiL+zcMfAUto%(ein*B-!w)tE7lK;AdrPIp^$Vs!tx}!nLzg~pbz~d`$@>mfey2?@ zwV&QyEH~EH%*TgGLt&my*NOCSr83cIhMT7}9o|2^-Ec3NY$V)^Tvr7Bq|0z@kyZ}1 z=J7t$$}wcf9ZdvTK+5S{TN`JV@*uiy(4A~+NsG~nItYygd9Z%$+uT(&m%$E>R4Dh< zR<<`C7UtcMsOhvTrN-{sw5Bs`0-g47wK`ZSl^d(8=EF+TjI!Fw{*))0P+pT~45Rrj zPe~nz`9SizK{Agq8YS~SlI@}5G|Q#N&g!({WZ^pPY-Mp!UMM$~m(4-iBY0fm=edbO z@Rm^*cxfVd^XwnUXkWaVciv1HUq4rBe7rm@`4kNyFnZ!?U&ys&6DSzQGU||M(=Zb3bm00 zr|$h_ptVEODoC$=J0~9}9YiRiSf@^@sf)gpW}axamq)BT z{jCEs;Z!}@OS?N)%2_@?#Tpa&KRKmY;|fB*y_009U<00Izz00jPf0!6c6 zmihd@VEt)Wf6)sT2tWV=5P$##AOHafKmY;|fB*#EN`YFTc+tv#;c?}>SzKMoegp8~ zR(pHv3jfHzU_CXgr`F$ZWdV_C2tWV=5P$##AOHafKmY;|fB*!B0=42rGyfW(P%W-j zvTp&>-~T@`tS7^uSb+cpAOHafKmY;|fB*y_009U%NwYsjf zA9RD07TrLo7kC}sNH-Kpc6Kz~a>k{*zl3zZ1it=nTE7_dzybjXKmY;|fB*y_009U< z00Izz!0ZL0g@IT3x6l7^|G(K+ERFyH2tWV=5P$##AOHafKmY;|n4>_^EE@In`u_{V zdNIeG;Ls3&00bZa0SG_<0uX=z1Rwwb2+T~NSgcpl>;KOT>)Ff>gM=Xf0SG_<0uX=z z1Rwwb2tWV=5SW+1Idh>>d1o=b{{P&tp3f^II4%Sr009U<00Izz00bZa0SG_<0y7j? zpf3QD>;K0286E{mLjVF0fB*y_009U<00Izz00bZ~cLBcskMsYzR|7Ht0SG_<0uX=z u1Rwwb2tWV=5SXa||NlSE|7S{!#329y2tWV=5P$##AOHafKmY=B7x)JjyMH+V literal 0 HcmV?d00001 diff --git a/259045253/test/blackbox/random_testing/coverage.txt b/259045253/test/blackbox/random_testing/coverage.txt new file mode 100644 index 00000000..dc8da91c --- /dev/null +++ b/259045253/test/blackbox/random_testing/coverage.txt @@ -0,0 +1,13 @@ +Name Stmts Miss Cover Missing +-------------------------------------------------------------------------------------------- +259045253/test/__init__.py 0 0 100% +259045253/test/blackbox/__init__.py 0 0 100% +259045253/test/blackbox/random_testing/__init__.py 0 0 100% +259045253/test/blackbox/random_testing/test_src_evaluator.py 6144 0 100% +src/__init__.py 49 40 18% 21-29, 33-89 +src/evaluator.py 338 259 23% 71-72, 76-77, 83, 89-94, 98-102, 106-110, 115, 118, 127, 134, 137-293, 297-304, 308-354, 357-358, 361-362, 365-370, 373-375, 378-381, 384-387, 392, 395-407, 410-418, 421-422, 425-426, 443, 445, 449 +src/lexer.py 197 180 9% 8-13, 16-21, 24-26, 29-30, 33-34, 37-51, 54-73, 76-88, 91-173, 176-198, 204-209, 212-240, 243 +src/parser.py 350 235 33% 17, 91-92, 97-99, 104, 118-120, 125, 142-146, 151-153, 156, 186-244, 247, 252, 257, 260-263, 266-280, 283-294, 297-300, 303-308, 311-313, 316-323, 326-329, 332-336, 341-359, 362-374, 377, 380, 383-387, 390-392, 395-397, 400-402, 405-407, 410-413, 416-426, 429-431, 434-440, 443-450, 453-466, 469-481, 484-486, 489-491, 494-504, 507-524, 529-535, 538-540 +src/tokens.py 48 0 100% +-------------------------------------------------------------------------------------------- +TOTAL 7126 714 90% diff --git a/259045253/test/blackbox/random_testing/pynguin-report/cov_report.html b/259045253/test/blackbox/random_testing/pynguin-report/cov_report.html new file mode 100644 index 00000000..f5edaf2e --- /dev/null +++ b/259045253/test/blackbox/random_testing/pynguin-report/cov_report.html @@ -0,0 +1,317 @@ + + + + + Pynguin coverage report + + + +

Pynguin coverage report for module 'src'

+

Achieved 100.00% branch coverage: +1/1 branchless code objects covered. +0/0 branches covered.

+ + + + + + + +
+ 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 + 11 + 12 + 13 + 14 + 15 + 16 + 17 + 18 + 19 + 20 + 21 + 22 + 23 + 24 + 25 + 26 + 27 + 28 + 29 + 30 + 31 + 32 + 33 + 34 + 35 + 36 + 37 + 38 + 39 + 40 + 41 + 42 + 43 + 44 + 45 + 46 + 47 + 48 + 49 + 50 + 51 + 52 + 53 + 54 + 55 + 56 + 57 + 58 + 59 + 60 + 61 + 62 + 63 + 64 + 65 + 66 + 67 + 68 + 69 + 70 + 71 + 72 + 73 + 74 + 75 + 76 + 77 + 78 + 79 + 80 + 81 + 82 + 83 + 84 + 85 + 86 + 87 + 88 + 
import argparse
+from argparse import ArgumentParser, RawDescriptionHelpFormatter
+from .lexer import Lexer
+from .parser import Parser
+from .evaluator import evaluate, setup_runtime
+import textwrap
+import code
+
+class Repl(code.InteractiveConsole): # pragma: no cover
+    env = {}
+
+    def runsource(self, source: str, filename="<input>", symbol="single"):
+        lexer = Lexer(source)
+        parser = Parser(lexer)
+        expressions = parser.run()
+        evaluate(expressions, self.env)
+
+
+def read_file_contents(filename: str) -> str | None: # pragma: no cover
+    if filename is None:
+        raise ValueError("filename cannot be None")
+    try:
+        with open(filename, "r") as file:
+            contents = file.read()
+        return contents
+    except FileNotFoundError:
+        print(f"error: file `{filename}` doesn't exist")
+        return None
+
+
+def main():# pragma: no cover
+    usage_message = "interpret [-h | --help] [--debug] [file] [-- arg]"
+    arg_parser = ArgumentParser(
+        usage=usage_message,
+        description=textwrap.dedent("""
+        An interpreter for simple programming language developed for Software Measurement and
+        Quality Assurance coursework.
+
+        Checkout https://github.com/JackyLeicester/Coursework for the syntax of the language.
+        """),
+        formatter_class=RawDescriptionHelpFormatter,
+        epilog=textwrap.dedent("""
+        Copyright © 2025
+        Jacky Liang Xu, Myrza Danike, Syed Fasiuddin, Vatsal Chaudhari."""),
+    )
+    arg_parser.add_argument("file", nargs="?", help=": program read from script file")
+    arg_parser.add_argument("--debug", action="store_true", help="enable debug output")
+    arg_parser.add_argument(
+        "args",
+        nargs=argparse.REMAINDER,
+        help=": additional argument after `--` passed through",
+    )
+    args = arg_parser.parse_args()
+
+    if not args.file:
+        repl = Repl()
+        repl.interact(banner="", exitmsg="")
+        exit(0)
+
+    contents = read_file_contents(args.file)
+    if not contents:
+        exit(1)
+
+    lexer = Lexer(contents)
+
+    # Debug: Will execute if run with "--debug" argument
+    if args.debug:
+        print("=== TOKENS ===")
+        token, token_str = lexer.next_token()
+        while token.name != "EOF":
+            print(f"{token.name:15} {token_str}")
+            token, token_str = lexer.next_token()
+        print(f"{token.name:15} {token_str}")
+
+    lexer = Lexer(contents)
+    parser = Parser(lexer)
+    expressions = parser.run()
+
+    # Since our language doesn't support arrays, we can only pass one argument
+    # to the script, if we add functionality of arrays then we can pass in
+    # `argc` and `argv` similar to how its done in C.
+    env = None
+    if len(args.args) > 1:
+        arg_parser.print_usage()
+        exit(1)
+    if len(args.args) == 1:
+        env = setup_runtime(args.args[0])
+    return evaluate(expressions, env)
+
+
+
+

Created at 2025-12-24 00:06:10.935990

+
+ + \ No newline at end of file diff --git a/259045253/test/blackbox/random_testing/pynguin-report/cov_report.xml b/259045253/test/blackbox/random_testing/pynguin-report/cov_report.xml new file mode 100644 index 00000000..7ed276e8 --- /dev/null +++ b/259045253/test/blackbox/random_testing/pynguin-report/cov_report.xml @@ -0,0 +1,17 @@ + + + src + + + + + + + + + + + + + + \ No newline at end of file diff --git a/259045253/test/blackbox/random_testing/pynguin-report/pynguin-cli-params.txt b/259045253/test/blackbox/random_testing/pynguin-report/pynguin-cli-params.txt new file mode 100644 index 00000000..d875e1cb --- /dev/null +++ b/259045253/test/blackbox/random_testing/pynguin-report/pynguin-cli-params.txt @@ -0,0 +1,290 @@ +--algorithm +RANDOM +--large_language_model.call_llm_for_uncovered_targets +False +--large_language_model.call_llm_on_stall_detection +False +--large_language_model.coverage_threshold +1 +--large_language_model.enable_response_caching +False +--large_language_model.hybrid_initial_population +False +--large_language_model.llm_test_case_percentage +0.5 +--large_language_model.max_llm_interventions +1 +--large_language_model.max_plateau_len +25 +--large_language_model.model_name +gpt-4o-mini +--large_language_model.temperature +0.8 +--local_search.local_search +True +--local_search.local_search_collections +False +--local_search.local_search_complex_objects +False +--local_search.local_search_different_datatype +False +--local_search.local_search_llm +False +--local_search.local_search_primitives +True +--local_search.local_search_probability +0.02 +--local_search.local_search_same_datatype +True +--local_search.local_search_time +5000 +--local_search.ls_dict_max_insertions +10 +--local_search.ls_different_type_collection_probability +0.3 +--local_search.ls_different_type_primitive_probability +0.3 +--local_search.ls_int_delta_increasing_factor +2 +--local_search.ls_llm_whole_module +False +--local_search.ls_max_different_type_mutations +10 +--local_search.ls_random_parametrized_statement_call_count +10 +--local_search.ls_string_random_mutation_count +10 +--mio.exploitation_starts_at_percent +0.5 +--mio.focused_config.number_of_mutations +10 +--mio.focused_config.number_of_tests_per_target +1 +--mio.focused_config.random_test_or_from_archive_probability +0.0 +--mio.initial_config.number_of_mutations +1 +--mio.initial_config.number_of_tests_per_target +10 +--mio.initial_config.random_test_or_from_archive_probability +0.5 +--module_name +src.evaluator +--project_path +. +--pynguinml.ignore_constraints_probability +0.25 +--pynguinml.max_ndim +4 +--pynguinml.max_shape_dim +4 +--pynguinml.ml_testing_enabled +False +--random.max_sequence_length +10 +--random.max_sequences_combined +10 +--search_algorithm.change_parameter_probability +0.1 +--search_algorithm.chop_max_length +True +--search_algorithm.chromosome_length +48 +--search_algorithm.crossover_rate +0.648 +--search_algorithm.elite +1 +--search_algorithm.filter_covered_targets_from_test_cluster +False +--search_algorithm.max_initial_tests +10 +--search_algorithm.min_initial_tests +1 +--search_algorithm.number_of_mutations +3 +--search_algorithm.population +10 +--search_algorithm.random_perturbation +0.2 +--search_algorithm.rank_bias +1.68 +--search_algorithm.selection +RANK_SELECTION +--search_algorithm.statement_insertion_probability +0.5 +--search_algorithm.test_change_probability +0.3333333333333333 +--search_algorithm.test_delete_probability +0.3333333333333333 +--search_algorithm.test_insert_probability +0.3333333333333333 +--search_algorithm.test_insertion_probability +0.1 +--search_algorithm.tournament_size +4 +--search_algorithm.use_archive +False +--seeding.constant_seeding +True +--seeding.dynamic_constant_seeding +True +--seeding.initial_population_mutations +0 +--seeding.initial_population_seeding +False +--seeding.max_dynamic_length +1000 +--seeding.max_dynamic_pool_size +50 +--seeding.seed +0 +--seeding.seed_from_archive +False +--seeding.seed_from_archive_mutations +3 +--seeding.seed_from_archive_probability +0.2 +--seeding.seeded_dynamic_values_reuse_probability +0.6 +--seeding.seeded_primitives_reuse_probability +0.2 +--seeding.seeded_testcases_reuse_probability +0.9 +--statistics_output.coverage_metrics +BRANCH +--statistics_output.create_coverage_report +False +--statistics_output.output_variables +TargetModule +Coverage +--statistics_output.report_dir +pynguin-report +--statistics_output.statistics_backend +CSV +--statistics_output.store_test_before_execution +False +--statistics_output.timeline_interpolation +True +--statistics_output.timeline_interval +1000000000 +--statistics_output.type_guess_top_n +10 +--stopping.maximum_coverage +100 +--stopping.maximum_coverage_plateau +-1 +--stopping.maximum_iterations +-1 +--stopping.maximum_memory +3000 +--stopping.maximum_search_time +60 +--stopping.maximum_slicing_time +600 +--stopping.maximum_statement_executions +-1 +--stopping.maximum_test_execution_timeout +5 +--stopping.maximum_test_executions +-1 +--stopping.minimum_coverage +100 +--stopping.minimum_plateau_iterations +-1 +--stopping.test_execution_time_per_statement +1 +--string_statement.faker_string_weight +0.3 +--string_statement.fandango_faker_string_weight +0.0 +--string_statement.fandango_string_weight +0.4 +--string_statement.random_string_weight +0.3 +--subprocess +False +--subprocess_if_recommended +True +--test_case_output.allow_stale_assertions +False +--test_case_output.assertion_generation +NONE +--test_case_output.export_strategy +PY_TEST +--test_case_output.float_precision +0.01 +--test_case_output.format_with_black +True +--test_case_output.max_length_test_case +2500 +--test_case_output.minimization.test_case_minimization_direction +BACKWARD +--test_case_output.minimization.test_case_minimization_strategy +CASE +--test_case_output.mutation_order +1 +--test_case_output.mutation_strategy +FIRST_ORDER_MUTANTS +--test_case_output.output_path +259045253/test/blackbox/random_testing +--test_case_output.post_process +True +--test_creation.any_weight +5 +--test_creation.bytes_length +20 +--test_creation.collection_size +5 +--test_creation.insertion_uut +0.5 +--test_creation.max_attempts +1000 +--test_creation.max_delta +20 +--test_creation.max_int +2048 +--test_creation.max_recursion +10 +--test_creation.max_size +100 +--test_creation.negate_type +0.1 +--test_creation.none_weight +1 +--test_creation.object_reuse_probability +0.9 +--test_creation.original_type_weight +5 +--test_creation.primitive_reuse_probability +0.5 +--test_creation.skip_optional_parameter_probability +0.7 +--test_creation.string_length +20 +--test_creation.type_tracing_kept_guesses +2 +--test_creation.type_tracing_weight +10 +--test_creation.use_random_object_for_call +0.1 +--test_creation.wrap_var_param_type_probability +0.7 +--to_cover.enable_inline_pragma_no_cover +True +--to_cover.enable_inline_pynguin_no_cover +True +--type_inference.subtype_inference +NONE +--type_inference.type_inference_strategy +TYPE_HINTS +--type_inference.type_tracing +0.0 +--type_inference.type_tracing_argument_type_weight +0.5 +--type_inference.type_tracing_attribute_weight +0.2 +--type_inference.type_tracing_subtype_weight +0.3 +--use_master_worker +True +-v \ No newline at end of file diff --git a/259045253/test/blackbox/random_testing/pynguin-report/pynguin-config.toml b/259045253/test/blackbox/random_testing/pynguin-report/pynguin-config.toml new file mode 100644 index 00000000..fb832e78 --- /dev/null +++ b/259045253/test/blackbox/random_testing/pynguin-report/pynguin-config.toml @@ -0,0 +1,192 @@ +project_path = "." +module_name = "src.evaluator" +algorithm = "RANDOM" +ignore_modules = [] +ignore_methods = [] +subprocess = false +subprocess_if_recommended = true +use_master_worker = true + +[test_case_output] +output_path = "259045253/test/blackbox/random_testing" +crash_path = "" +export_strategy = "PY_TEST" +max_length_test_case = 2500 +assertion_generation = "NONE" +allow_stale_assertions = false +mutation_strategy = "FIRST_ORDER_MUTANTS" +mutation_order = 1 +post_process = true +float_precision = 0.01 +format_with_black = true + +[statistics_output] +report_dir = "pynguin-report" +statistics_backend = "CSV" +timeline_interval = 1000000000 +timeline_interpolation = true +coverage_metrics = [ "BRANCH",] +output_variables = [ "TargetModule", "Coverage",] +configuration_id = "" +run_id = "" +project_name = "" +create_coverage_report = false +type_guess_top_n = 10 +store_test_before_execution = false + +[stopping] +maximum_search_time = 60 +maximum_test_executions = -1 +maximum_statement_executions = -1 +maximum_slicing_time = 600 +maximum_iterations = -1 +maximum_test_execution_timeout = 5 +maximum_coverage = 100 +maximum_coverage_plateau = -1 +minimum_coverage = 100 +minimum_plateau_iterations = -1 +maximum_memory = 3000 +test_execution_time_per_statement = 1 + +[large_language_model] +api_key = "" +model_name = "gpt-4o-mini" +temperature = 0.8 +hybrid_initial_population = false +llm_test_case_percentage = 0.5 +enable_response_caching = false +call_llm_for_uncovered_targets = false +coverage_threshold = 1 +call_llm_on_stall_detection = false +max_plateau_len = 25 +max_llm_interventions = 1 + +[string_statement] +random_string_weight = 0.3 +faker_string_weight = 0.3 +fandango_string_weight = 0.4 +fandango_faker_string_weight = 0.0 + +[seeding] +seed = 0 +constant_seeding = true +initial_population_seeding = false +initial_population_data = "" +seeded_testcases_reuse_probability = 0.9 +initial_population_mutations = 0 +dynamic_constant_seeding = true +seeded_primitives_reuse_probability = 0.2 +seeded_dynamic_values_reuse_probability = 0.6 +seed_from_archive = false +seed_from_archive_probability = 0.2 +seed_from_archive_mutations = 3 +max_dynamic_length = 1000 +max_dynamic_pool_size = 50 + +[type_inference] +type_inference_strategy = "TYPE_HINTS" +type_tracing = 0.0 +subtype_inference = "NONE" +type_tracing_subtype_weight = 0.3 +type_tracing_argument_type_weight = 0.5 +type_tracing_attribute_weight = 0.2 + +[pynguinml] +ml_testing_enabled = false +constraints_path = "" +dtype_mapping_path = "" +constructor_function = "" +constructor_function_parameter = "" +max_ndim = 4 +max_shape_dim = 4 +ignore_constraints_probability = 0.25 + +[test_creation] +max_recursion = 10 +max_delta = 20 +max_int = 2048 +string_length = 20 +bytes_length = 20 +collection_size = 5 +primitive_reuse_probability = 0.5 +object_reuse_probability = 0.9 +none_weight = 1 +any_weight = 5 +original_type_weight = 5 +type_tracing_weight = 10 +type_tracing_kept_guesses = 2 +wrap_var_param_type_probability = 0.7 +negate_type = 0.1 +skip_optional_parameter_probability = 0.7 +max_attempts = 1000 +insertion_uut = 0.5 +max_size = 100 +use_random_object_for_call = 0.1 + +[search_algorithm] +min_initial_tests = 1 +max_initial_tests = 10 +population = 10 +chromosome_length = 48 +chop_max_length = true +elite = 1 +crossover_rate = 0.648 +test_insertion_probability = 0.1 +test_delete_probability = 0.3333333333333333 +test_change_probability = 0.3333333333333333 +test_insert_probability = 0.3333333333333333 +statement_insertion_probability = 0.5 +random_perturbation = 0.2 +change_parameter_probability = 0.1 +tournament_size = 4 +rank_bias = 1.68 +selection = "RANK_SELECTION" +use_archive = false +filter_covered_targets_from_test_cluster = false +number_of_mutations = 3 + +[mio] +exploitation_starts_at_percent = 0.5 + +[random] +max_sequence_length = 10 +max_sequences_combined = 10 + +[to_cover] +only_cover = [] +no_cover = [] +enable_inline_pynguin_no_cover = true +enable_inline_pragma_no_cover = true + +[local_search] +local_search = true +local_search_same_datatype = true +local_search_different_datatype = false +local_search_llm = false +local_search_primitives = true +local_search_collections = false +local_search_complex_objects = false +local_search_probability = 0.02 +local_search_time = 5000 +ls_int_delta_increasing_factor = 2 +ls_string_random_mutation_count = 10 +ls_random_parametrized_statement_call_count = 10 +ls_max_different_type_mutations = 10 +ls_different_type_primitive_probability = 0.3 +ls_different_type_collection_probability = 0.3 +ls_dict_max_insertions = 10 +ls_llm_whole_module = false + +[test_case_output.minimization] +test_case_minimization_strategy = "CASE" +test_case_minimization_direction = "BACKWARD" + +[mio.initial_config] +number_of_tests_per_target = 10 +random_test_or_from_archive_probability = 0.5 +number_of_mutations = 1 + +[mio.focused_config] +number_of_tests_per_target = 1 +random_test_or_from_archive_probability = 0.0 +number_of_mutations = 10 diff --git a/259045253/test/blackbox/random_testing/pynguin-report/pynguin-config.txt b/259045253/test/blackbox/random_testing/pynguin-report/pynguin-config.txt new file mode 100644 index 00000000..33ae89fe --- /dev/null +++ b/259045253/test/blackbox/random_testing/pynguin-report/pynguin-config.txt @@ -0,0 +1,88 @@ +("Configuration(project_path='.', module_name='src.evaluator', " + "test_case_output=TestCaseOutputConfiguration(output_path='259045253/test/blackbox/random_testing', " + "crash_path='', export_strategy=, " + 'max_length_test_case=2500, assertion_generation=, allow_stale_assertions=False, " + 'mutation_strategy=, mutation_order=1, post_process=True, " + 'minimization=Minimization(test_case_minimization_strategy=, test_case_minimization_direction=), float_precision=0.01, format_with_black=True), " + "algorithm=, " + "statistics_output=StatisticsOutputConfiguration(report_dir='pynguin-report', " + "statistics_backend=, " + 'timeline_interval=1000000000, timeline_interpolation=True, ' + "coverage_metrics=[], " + "output_variables=[TargetModule, Coverage], configuration_id='', run_id='', " + "project_name='', create_coverage_report=False, type_guess_top_n=10, " + 'store_test_before_execution=False), ' + 'stopping=StoppingConfiguration(maximum_search_time=60, ' + 'maximum_test_executions=-1, maximum_statement_executions=-1, ' + 'maximum_slicing_time=600, maximum_iterations=-1, ' + 'maximum_test_execution_timeout=5, maximum_coverage=100, ' + 'maximum_coverage_plateau=-1, minimum_coverage=100, ' + 'minimum_plateau_iterations=-1, maximum_memory=3000, ' + 'test_execution_time_per_statement=1), ' + "large_language_model=LLMConfiguration(api_key='', model_name='gpt-4o-mini', " + 'temperature=0.8, hybrid_initial_population=False, ' + 'llm_test_case_percentage=0.5, enable_response_caching=False, ' + 'call_llm_for_uncovered_targets=False, coverage_threshold=1, ' + 'call_llm_on_stall_detection=False, max_plateau_len=25, ' + 'max_llm_interventions=1), ' + 'string_statement=StringStatementConfiguration(random_string_weight=0.3, ' + 'faker_string_weight=0.3, fandango_string_weight=0.4, ' + 'fandango_faker_string_weight=0.0), seeding=SeedingConfiguration(seed=0, ' + 'constant_seeding=True, initial_population_seeding=False, ' + "initial_population_data='', seeded_testcases_reuse_probability=0.9, " + 'initial_population_mutations=0, dynamic_constant_seeding=True, ' + 'seeded_primitives_reuse_probability=0.2, ' + 'seeded_dynamic_values_reuse_probability=0.6, seed_from_archive=False, ' + 'seed_from_archive_probability=0.2, seed_from_archive_mutations=3, ' + 'max_dynamic_length=1000, max_dynamic_pool_size=50), ' + 'type_inference=TypeInferenceConfiguration(type_inference_strategy=, type_tracing=0.0, " + "subtype_inference=, " + 'type_tracing_subtype_weight=0.3, type_tracing_argument_type_weight=0.5, ' + 'type_tracing_attribute_weight=0.2), ' + 'pynguinml=PynguinMLConfiguration(ml_testing_enabled=False, ' + "constraints_path='', dtype_mapping_path='', constructor_function='', " + "constructor_function_parameter='', max_ndim=4, max_shape_dim=4, " + 'ignore_constraints_probability=0.25), ' + 'test_creation=TestCreationConfiguration(max_recursion=10, max_delta=20, ' + 'max_int=2048, string_length=20, bytes_length=20, collection_size=5, ' + 'primitive_reuse_probability=0.5, object_reuse_probability=0.9, ' + 'none_weight=1, any_weight=5, original_type_weight=5, type_tracing_weight=10, ' + 'type_tracing_kept_guesses=2, wrap_var_param_type_probability=0.7, ' + 'negate_type=0.1, skip_optional_parameter_probability=0.7, max_attempts=1000, ' + 'insertion_uut=0.5, max_size=100, use_random_object_for_call=0.1), ' + 'search_algorithm=SearchAlgorithmConfiguration(min_initial_tests=1, ' + 'max_initial_tests=10, population=10, chromosome_length=48, ' + 'chop_max_length=True, elite=1, crossover_rate=0.648, ' + 'test_insertion_probability=0.1, test_delete_probability=0.3333333333333333, ' + 'test_change_probability=0.3333333333333333, ' + 'test_insert_probability=0.3333333333333333, ' + 'statement_insertion_probability=0.5, random_perturbation=0.2, ' + 'change_parameter_probability=0.1, tournament_size=4, rank_bias=1.68, ' + "selection=, use_archive=False, " + 'filter_covered_targets_from_test_cluster=False, number_of_mutations=3), ' + 'mio=MIOConfiguration(initial_config=MIOPhaseConfiguration(number_of_tests_per_target=10, ' + 'random_test_or_from_archive_probability=0.5, number_of_mutations=1), ' + 'focused_config=MIOPhaseConfiguration(number_of_tests_per_target=1, ' + 'random_test_or_from_archive_probability=0.0, number_of_mutations=10), ' + 'exploitation_starts_at_percent=0.5), ' + 'random=RandomConfiguration(max_sequence_length=10, ' + 'max_sequences_combined=10), to_cover=ToCoverConfiguration(only_cover=[], ' + 'no_cover=[], enable_inline_pynguin_no_cover=True, ' + 'enable_inline_pragma_no_cover=True), ignore_modules=[], ignore_methods=[], ' + 'subprocess=False, subprocess_if_recommended=True, ' + 'local_search=LocalSearchConfiguration(local_search=True, ' + 'local_search_same_datatype=True, local_search_different_datatype=False, ' + 'local_search_llm=False, local_search_primitives=True, ' + 'local_search_collections=False, local_search_complex_objects=False, ' + 'local_search_probability=0.02, local_search_time=5000, ' + 'ls_int_delta_increasing_factor=2, ls_string_random_mutation_count=10, ' + 'ls_random_parametrized_statement_call_count=10, ' + 'ls_max_different_type_mutations=10, ' + 'ls_different_type_primitive_probability=0.3, ' + 'ls_different_type_collection_probability=0.3, ls_dict_max_insertions=10, ' + 'ls_llm_whole_module=False), use_master_worker=True)') \ No newline at end of file diff --git a/259045253/test/blackbox/random_testing/pynguin-report/signatures/evaluator/evaluator_result.json b/259045253/test/blackbox/random_testing/pynguin-report/signatures/evaluator/evaluator_result.json new file mode 100644 index 00000000..bfda5bc0 --- /dev/null +++ b/259045253/test/blackbox/random_testing/pynguin-report/signatures/evaluator/evaluator_result.json @@ -0,0 +1 @@ +[{"file": "evaluator.py", "line_number": 40, "col_offset": 24, "type": [], "function": "_ReturnSignal.__init__", "parameter": "value"}, {"file": "evaluator.py", "line_number": 49, "col_offset": 24, "type": ["int"], "function": "_ExitSignal.__init__", "parameter": "code"}, {"file": "evaluator.py", "line_number": 61, "col_offset": 19, "type": ["str"], "function": "setup_runtime", "parameter": "arg"}, {"file": "evaluator.py", "line_number": 61, "col_offset": 5, "type": ["list"], "function": "setup_runtime"}, {"file": "evaluator.py", "line_number": 433, "col_offset": 14, "type": ["list"], "function": "evaluate", "parameter": "expressions"}, {"file": "evaluator.py", "line_number": 433, "col_offset": 45, "type": ["None", "list"], "function": "evaluate", "parameter": "env"}, {"file": "evaluator.py", "line_number": 433, "col_offset": 5, "type": ["Any"], "function": "evaluate"}] \ No newline at end of file diff --git a/259045253/test/blackbox/random_testing/pynguin-report/signatures/lexer/lexer_result.json b/259045253/test/blackbox/random_testing/pynguin-report/signatures/lexer/lexer_result.json new file mode 100644 index 00000000..148e87d0 --- /dev/null +++ b/259045253/test/blackbox/random_testing/pynguin-report/signatures/lexer/lexer_result.json @@ -0,0 +1 @@ +[{"file": "lexer.py", "line_number": 7, "col_offset": 24, "type": ["str"], "function": "Lexer.__init__", "parameter": "input"}, {"file": "lexer.py", "line_number": 233, "col_offset": 5, "type": [], "function": "Lexer.__repr__"}, {"file": "lexer.py", "line_number": 90, "col_offset": 5, "type": ["tuple"], "function": "Lexer.next_token"}, {"file": "lexer.py", "line_number": 23, "col_offset": 5, "type": [], "function": "Lexer.peek"}, {"file": "lexer.py", "line_number": 15, "col_offset": 5, "type": ["None"], "function": "Lexer.read_char"}, {"file": "lexer.py", "line_number": 53, "col_offset": 5, "type": ["tuple"], "function": "Lexer.read_number"}, {"file": "lexer.py", "line_number": 36, "col_offset": 5, "type": [], "function": "Lexer.skip_non_tokens"}, {"file": "lexer.py", "line_number": 32, "col_offset": 5, "type": [], "function": "Lexer.skip_singleline_comment"}, {"file": "lexer.py", "line_number": 28, "col_offset": 5, "type": [], "function": "Lexer.skip_whitespace"}] \ No newline at end of file diff --git a/259045253/test/blackbox/random_testing/pynguin-report/signatures/src/src_result.json b/259045253/test/blackbox/random_testing/pynguin-report/signatures/src/src_result.json new file mode 100644 index 00000000..b09b6691 --- /dev/null +++ b/259045253/test/blackbox/random_testing/pynguin-report/signatures/src/src_result.json @@ -0,0 +1 @@ +[{"file": "src.py", "line_number": 12, "col_offset": 25, "type": ["str"], "function": "Repl.runsource", "parameter": "source"}, {"file": "src.py", "line_number": 12, "col_offset": 38, "type": [], "function": "Repl.runsource", "parameter": "filename"}, {"file": "src.py", "line_number": 12, "col_offset": 58, "type": [], "function": "Repl.runsource", "parameter": "symbol"}, {"file": "src.py", "line_number": 12, "col_offset": 5, "type": [], "function": "Repl.runsource"}, {"file": "src.py", "line_number": 19, "col_offset": 24, "type": ["str"], "function": "read_file_contents", "parameter": "filename"}, {"file": "src.py", "line_number": 19, "col_offset": 5, "type": ["None", "str"], "function": "read_file_contents"}] \ No newline at end of file diff --git a/259045253/test/blackbox/random_testing/pynguin-report/statistics.csv b/259045253/test/blackbox/random_testing/pynguin-report/statistics.csv new file mode 100644 index 00000000..f5a66e27 --- /dev/null +++ b/259045253/test/blackbox/random_testing/pynguin-report/statistics.csv @@ -0,0 +1,15 @@ +"TargetModule","Coverage" +"foo","1.0" +"foo","1.0" +"foo","1.0" +"src.lexer","1.0" +"src.lexer","1.0" +"src.lexer","1.0" +"src.lexer","1.0" +"src.lexer","0.35119047619047616" +"src","1.0" +"src","1.0" +"src.evaluator","0.17482517482517482" +"src.evaluator","0.17482517482517482" +"src.evaluator","0.17482517482517482" +"src.evaluator","0.17482517482517482" diff --git a/259045253/test/blackbox/random_testing/test_src_evaluator.py b/259045253/test/blackbox/random_testing/test_src_evaluator.py new file mode 100644 index 00000000..1b986db1 --- /dev/null +++ b/259045253/test/blackbox/random_testing/test_src_evaluator.py @@ -0,0 +1,7946 @@ +# Test cases automatically generated by Pynguin (https://www.pynguin.eu). +# Please check them before you use them. +import src.evaluator as module_0 +import src.parser as module_1 +import builtins as module_2 + + +def test_case_0(): + str_0 = "JTE" + module_0.setup_runtime(str_0) + + +def test_case_1(): + str_0 = "JTE" + module_0.setup_runtime(str_0) + list_0 = [] + module_0.evaluate(list_0) + + +def test_case_2(): + str_0 = "JTE" + module_0.setup_runtime(str_0) + list_0 = [] + module_0.evaluate(list_0) + + +def test_case_3(): + str_0 = "JTE" + module_0.setup_runtime(str_0) + + +def test_case_4(): + str_0 = "JTE" + module_0.setup_runtime(str_0) + list_0 = [] + module_0.evaluate(list_0) + + +def test_case_5(): + str_0 = "JTE" + module_0.setup_runtime(str_0) + list_0 = [] + module_0.evaluate(list_0) + + +def test_case_6(): + str_0 = "JTE" + module_0.setup_runtime(str_0) + list_0 = [] + var_0 = module_0.evaluate(list_0) + module_0._ReturnSignal(var_0) + + +def test_case_7(): + str_0 = "JTE" + module_0.setup_runtime(str_0) + + +def test_case_8(): + str_0 = "JTE" + module_0.setup_runtime(str_0) + list_0 = [] + module_0.evaluate(list_0) + + +def test_case_9(): + str_0 = "JTE" + module_0.setup_runtime(str_0) + list_0 = [] + module_0.evaluate(list_0) + + +def test_case_10(): + str_0 = "JTE" + module_0.setup_runtime(str_0) + list_0 = [] + module_0.evaluate(list_0) + + +def test_case_11(): + str_0 = "JTE" + module_0.setup_runtime(str_0) + list_0 = [] + module_0.evaluate(list_0) + + +def test_case_12(): + str_0 = "JTE" + module_0.setup_runtime(str_0) + list_0 = [] + module_0.evaluate(list_0) + module_0._ReturnSignal(list_0) + + +def test_case_13(): + str_0 = "JTE" + module_0.setup_runtime(str_0) + list_0 = [] + module_0.evaluate(list_0) + + +def test_case_14(): + str_0 = "JTE" + module_0.setup_runtime(str_0) + list_0 = [] + module_0.evaluate(list_0) + none_type_0 = None + module_0._ReturnSignal(none_type_0) + + +def test_case_15(): + str_0 = "JTE" + module_0.setup_runtime(str_0) + list_0 = [] + module_0.evaluate(list_0) + break_signal_0 = module_0._BreakSignal() + module_0._ReturnSignal(break_signal_0) + + +def test_case_16(): + str_0 = "JTE" + module_0.setup_runtime(str_0) + + +def test_case_17(): + str_0 = "JTE" + module_0.setup_runtime(str_0) + list_0 = [] + module_0.evaluate(list_0) + + +def test_case_18(): + str_0 = "JTE" + module_0.setup_runtime(str_0) + list_0 = [] + module_0.evaluate(list_0) + continue_signal_0 = module_0._ContinueSignal() + module_0._ReturnSignal(continue_signal_0) + + +def test_case_19(): + str_0 = '"Ow' + module_0.setup_runtime(str_0) + + +def test_case_20(): + str_0 = "JTE" + module_0.setup_runtime(str_0) + list_0 = [] + module_0.evaluate(list_0) + + +def test_case_21(): + str_0 = '"Ow' + module_0.setup_runtime(str_0) + list_0 = [] + module_0.evaluate(list_0) + + +def test_case_22(): + str_0 = "JTE" + module_0.setup_runtime(str_0) + list_0 = [] + module_0.evaluate(list_0) + + +def test_case_23(): + str_0 = '"Ow' + module_0.setup_runtime(str_0) + list_0 = [] + var_0 = module_0.evaluate(list_0) + module_0._ReturnSignal(var_0) + + +def test_case_24(): + str_0 = "JTE" + module_0.setup_runtime(str_0) + list_0 = [] + module_0.evaluate(list_0) + + +def test_case_25(): + str_0 = "JTE" + module_0.setup_runtime(str_0) + bool_0 = False + module_0._ReturnSignal(bool_0) + + +def test_case_26(): + str_0 = "JTE" + module_0.setup_runtime(str_0) + list_0 = [] + module_0.evaluate(list_0) + + +def test_case_27(): + str_0 = '"Ow' + module_0.setup_runtime(str_0) + list_0 = [] + module_0.evaluate(list_0) + + +def test_case_28(): + str_0 = '"Ow' + module_0.setup_runtime(str_0) + list_0 = [] + module_0.evaluate(list_0) + + +def test_case_29(): + str_0 = "JTE" + module_0.setup_runtime(str_0) + + +def test_case_30(): + str_0 = '"Ow' + module_0.setup_runtime(str_0) + list_0 = [] + module_0.evaluate(list_0) + + +def test_case_31(): + str_0 = "JTE" + module_0.setup_runtime(str_0) + list_0 = [] + module_0.evaluate(list_0) + + +def test_case_32(): + str_0 = "Number of parameters passed is not equal to number of function parameters" + module_0.setup_runtime(str_0) + + +def test_case_33(): + list_0 = [] + module_0.evaluate(list_0) + str_0 = "JTE" + list_1 = module_0.setup_runtime(str_0) + module_0._ReturnSignal(list_1) + + +def test_case_34(): + str_0 = "Number of parameters passed is not equal to number of function parameters" + module_0.setup_runtime(str_0) + list_0 = [] + module_0.evaluate(list_0) + + +def test_case_35(): + str_0 = "Number of parameters passed is not equal to number of function parameters" + module_0.setup_runtime(str_0) + + +def test_case_36(): + str_0 = " Any: if isinstance(node, CallExpression): name = node.identifier_name - args = [_eval(arg, env) for arg in node.parameters] + args = [] + for arg in node.parameters: + args.append(_eval(arg, env)) + if name == "sqrt": if len(args) != 1: raise RuntimeEvaluationError("sqrt expects 1 arguments")