[End to End Testing] Implement must change.

README.md

@@ -1,7 +1,7 @@
-[![Flake8](https://img.shields.io/badge/Flake8-passed-brightgreen)](https://github.com/RuminantFarmSystems/MASM/actions/workflows/combined_format_lint_test_mypy.yml)
+[![Flake8](https://img.shields.io/badge/Flake8-failed-red)](https://github.com/RuminantFarmSystems/MASM/actions/workflows/combined_format_lint_test_mypy.yml)
 [![Pytest](https://img.shields.io/badge/Pytest-passed-brightgreen)](https://github.com/RuminantFarmSystems/MASM/actions/workflows/combined_format_lint_test_mypy.yml)
 [![Coverage](https://img.shields.io/badge/Coverage-99%25-brightgreen)](https://github.com/RuminantFarmSystems/MASM/actions/workflows/combined_format_lint_test_mypy.yml)
-[![Mypy](https://img.shields.io/badge/Mypy-1135%20errors-red)](https://github.com/RuminantFarmSystems/MASM/actions/workflows/combined_format_lint_test_mypy.yml)
+[![Mypy](https://img.shields.io/badge/Mypy-1134%20errors-red)](https://github.com/RuminantFarmSystems/MASM/actions/workflows/combined_format_lint_test_mypy.yml)
 
 
 # RuFaS: Ruminant Farm Systems

RUFAS/input/metadata/properties/default.json

@@ -6349,6 +6349,11 @@
             "description": "The relative tolerance (expressed as a percent) used to compare expected and actual results for changed values found in the e2e results comparison.",
             "default": 0.0,
             "minimum": 0.0
+          },
+          "must_change_variables_path": {
+            "type": "string",
+            "description": "Path to the file listing the variables that are expected to differ from the recorded expected results for this input set. Each listed variable must differ from its expected value beyond the domain tolerance for the end-to-end test to pass.",
+            "default": ""
           }
         }
       },
@@ -6373,6 +6378,11 @@
             "description": "The relative tolerance (expressed as a percent) used to compare expected and actual results for changed values found in the e2e results comparison.",
             "default": 0.0,
             "minimum": 0.0
+          },
+          "must_change_variables_path": {
+            "type": "string",
+            "description": "Path to the file listing the variables that are expected to differ from the recorded expected results for this input set. Each listed variable must differ from its expected value beyond the domain tolerance for the end-to-end test to pass.",
+            "default": ""
           }
         }
       },
@@ -6397,6 +6407,11 @@
             "description": "The relative tolerance (expressed as a percent) used to compare expected and actual results for changed values found in the e2e results comparison.",
             "default": 0.0,
             "minimum": 0.0
+          },
+          "must_change_variables_path": {
+            "type": "string",
+            "description": "Path to the file listing the variables that are expected to differ from the recorded expected results for this input set. Each listed variable must differ from its expected value beyond the domain tolerance for the end-to-end test to pass.",
+            "default": ""
           }
         }
       },
@@ -6421,6 +6436,11 @@
             "description": "The relative tolerance (expressed as a percent) used to compare expected and actual results for changed values found in the e2e results comparison.",
             "default": 0.0,
             "minimum": 0.0
+          },
+          "must_change_variables_path": {
+            "type": "string",
+            "description": "Path to the file listing the variables that are expected to differ from the recorded expected results for this input set. Each listed variable must differ from its expected value beyond the domain tolerance for the end-to-end test to pass.",
+            "default": ""
           }
         }
       }

input/data/end_to_end_testing/animals_only_must_change_variables.json

@@ -0,0 +1,4 @@
+{
+    "description": "Variables listed in 'must_change_variables' are expected to differ from the recorded expected results, e.g. because of a known model change whose new values are not yet known. During an end-to-end testing run, each listed variable must differ from its recorded expected value beyond the domain tolerance: a matching value is reported as a failure, and differences in listed variables are not. Variable names must exactly match keys in the 'expected_results' of this input set's e2e_json_*_filter.json files; the 'changed_variables' list in the end-to-end comparison results shows which variables currently differ. Running the UPDATE_E2E_TEST_RESULTS task empties this list after regenerating the expected results.",
+    "must_change_variables": []
+}

input/data/end_to_end_testing/end_to_end_testing_result_paths.json

@@ -5,79 +5,91 @@
         "domain": "CropAndSoil",
         "expected_results_path": "input/data/end_to_end_testing/freestall/e2e_json_crop_soil_filter.json",
         "actual_results_path": "freestall_e2e_saved_variables_e2e_crop_and_soil_",
-        "tolerance": 0.1
+        "tolerance": 0.1,
+        "must_change_variables_path": "input/data/end_to_end_testing/freestall_must_change_variables.json"
       },
       {
         "domain": "Animal",
         "expected_results_path": "input/data/end_to_end_testing/freestall/e2e_json_animal_filter.json",
         "actual_results_path": "freestall_e2e_saved_variables_e2e_animal_",
-        "tolerance": 0.1
+        "tolerance": 0.1,
+        "must_change_variables_path": "input/data/end_to_end_testing/freestall_must_change_variables.json"
       },
       {
         "domain": "Manure",
         "expected_results_path": "input/data/end_to_end_testing/freestall/e2e_json_manure_filter.json",
         "actual_results_path": "freestall_e2e_saved_variables_e2e_manure_",
-        "tolerance": 0.1
+        "tolerance": 0.1,
+        "must_change_variables_path": "input/data/end_to_end_testing/freestall_must_change_variables.json"
       },
       {
         "domain": "Feed",
         "expected_results_path": "input/data/end_to_end_testing/freestall/e2e_json_feed_filter.json",
         "actual_results_path": "freestall_e2e_saved_variables_e2e_feed_",
-        "tolerance": 0.1
+        "tolerance": 0.1,
+        "must_change_variables_path": "input/data/end_to_end_testing/freestall_must_change_variables.json"
       }
     ],
     "open_lot_e2e": [
       {
         "domain": "CropAndSoil",
         "expected_results_path": "input/data/end_to_end_testing/open_lot/e2e_json_crop_soil_filter.json",
         "actual_results_path": "open_lot_e2e_saved_variables_e2e_crop_and_soil_",
-        "tolerance": 0.1
+        "tolerance": 0.1,
+        "must_change_variables_path": "input/data/end_to_end_testing/open_lot_must_change_variables.json"
       },
       {
         "domain": "Animal",
         "expected_results_path": "input/data/end_to_end_testing/open_lot/e2e_json_animal_filter.json",
         "actual_results_path": "open_lot_e2e_saved_variables_e2e_animal_",
-        "tolerance": 0.1
+        "tolerance": 0.1,
+        "must_change_variables_path": "input/data/end_to_end_testing/open_lot_must_change_variables.json"
       },
       {
         "domain": "Manure",
         "expected_results_path": "input/data/end_to_end_testing/open_lot/e2e_json_manure_filter.json",
         "actual_results_path": "open_lot_e2e_saved_variables_e2e_manure_",
-        "tolerance": 0.1
+        "tolerance": 0.1,
+        "must_change_variables_path": "input/data/end_to_end_testing/open_lot_must_change_variables.json"
       },
       {
         "domain": "Feed",
         "expected_results_path": "input/data/end_to_end_testing/open_lot/e2e_json_feed_filter.json",
         "actual_results_path": "open_lot_e2e_saved_variables_e2e_feed_",
-        "tolerance": 0.1
+        "tolerance": 0.1,
+        "must_change_variables_path": "input/data/end_to_end_testing/open_lot_must_change_variables.json"
       }
     ],
     "field_and_feed_e2e": [
       {
         "domain": "CropAndSoil",
         "expected_results_path": "input/data/end_to_end_testing/field_and_feed/e2e_json_crop_soil_filter.json",
         "actual_results_path": "field_and_feed_e2e_saved_variables_e2e_crop_and_soil_",
-        "tolerance": 0.1
+        "tolerance": 0.1,
+        "must_change_variables_path": "input/data/end_to_end_testing/field_and_feed_must_change_variables.json"
       },
       {
         "domain": "Manure",
         "expected_results_path": "input/data/end_to_end_testing/field_and_feed/e2e_json_manure_filter.json",
         "actual_results_path": "field_and_feed_e2e_saved_variables_e2e_manure_",
-        "tolerance": 0.1
+        "tolerance": 0.1,
+        "must_change_variables_path": "input/data/end_to_end_testing/field_and_feed_must_change_variables.json"
       },
       {
         "domain": "Feed",
         "expected_results_path": "input/data/end_to_end_testing/field_and_feed/e2e_json_feed_filter.json",
         "actual_results_path": "field_and_feed_e2e_saved_variables_e2e_feed_",
-        "tolerance": 0.1
+        "tolerance": 0.1,
+        "must_change_variables_path": "input/data/end_to_end_testing/field_and_feed_must_change_variables.json"
       }
     ],
     "animals_only_e2e": [
       {
         "domain": "Animal",
         "expected_results_path": "input/data/end_to_end_testing/animals_only/e2e_json_animal_filter.json",
         "actual_results_path": "animals_only_e2e_saved_variables_e2e_animal_",
-        "tolerance": 0.1
+        "tolerance": 0.1,
+        "must_change_variables_path": "input/data/end_to_end_testing/animals_only_must_change_variables.json"
       }
     ]
   }

input/data/end_to_end_testing/field_and_feed_must_change_variables.json

@@ -0,0 +1,4 @@
+{
+    "description": "Variables listed in 'must_change_variables' are expected to differ from the recorded expected results, e.g. because of a known model change whose new values are not yet known. During an end-to-end testing run, each listed variable must differ from its recorded expected value beyond the domain tolerance: a matching value is reported as a failure, and differences in listed variables are not. Variable names must exactly match keys in the 'expected_results' of this input set's e2e_json_*_filter.json files; the 'changed_variables' list in the end-to-end comparison results shows which variables currently differ. Running the UPDATE_E2E_TEST_RESULTS task empties this list after regenerating the expected results.",
+    "must_change_variables": []
+}

input/data/end_to_end_testing/freestall_must_change_variables.json

@@ -0,0 +1,4 @@
+{
+    "description": "Variables listed in 'must_change_variables' are expected to differ from the recorded expected results, e.g. because of a known model change whose new values are not yet known. During an end-to-end testing run, each listed variable must differ from its recorded expected value beyond the domain tolerance: a matching value is reported as a failure, and differences in listed variables are not. Variable names must exactly match keys in the 'expected_results' of this input set's e2e_json_*_filter.json files; the 'changed_variables' list in the end-to-end comparison results shows which variables currently differ. Running the UPDATE_E2E_TEST_RESULTS task empties this list after regenerating the expected results.",
+    "must_change_variables": []
+}

input/data/end_to_end_testing/open_lot_must_change_variables.json

@@ -0,0 +1,4 @@
+{
+    "description": "Variables listed in 'must_change_variables' are expected to differ from the recorded expected results, e.g. because of a known model change whose new values are not yet known. During an end-to-end testing run, each listed variable must differ from its recorded expected value beyond the domain tolerance: a matching value is reported as a failure, and differences in listed variables are not. Variable names must exactly match keys in the 'expected_results' of this input set's e2e_json_*_filter.json files; the 'changed_variables' list in the end-to-end comparison results shows which variables currently differ. Running the UPDATE_E2E_TEST_RESULTS task empties this list after regenerating the expected results.",
+    "must_change_variables": []
+}