From acb3ca1a01b445aeb356a57e12bc4527cd5fef3f Mon Sep 17 00:00:00 2001 From: Allister Liu Date: Wed, 24 Jun 2026 13:07:16 -0400 Subject: [PATCH 1/9] wip --- RUFAS/EEE/economics/preprocessing.py | 254 +++++++++++++++++- .../test_EEE/test_economics_preprocessing.py | 214 ++++++++++++++- 2 files changed, 463 insertions(+), 5 deletions(-) diff --git a/RUFAS/EEE/economics/preprocessing.py b/RUFAS/EEE/economics/preprocessing.py index 5b718af977..b13aeea8aa 100644 --- a/RUFAS/EEE/economics/preprocessing.py +++ b/RUFAS/EEE/economics/preprocessing.py @@ -17,7 +17,9 @@ import math import re from dataclasses import dataclass -from typing import Any, Dict, Iterable, List, Set +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Iterable, List, Set, Tuple from RUFAS.input_manager import InputManager from RUFAS.output_manager import OutputManager @@ -633,6 +635,251 @@ def _fetch_prices_with_exact_matches( prices[option] = price_data return prices + _CROP_TO_SEED_KEY: Dict[str, str] = { + "corn_grain": "commodity_prices_corn_seed_dollar_per_square_meter", + "corn_silage": "commodity_prices_corn_seed_dollar_per_square_meter", + "soybean_grain": "commodity_prices_soybean_seed_dollar_per_square_meter", + "soybean_hay": "commodity_prices_soybean_seed_dollar_per_square_meter", + "winter_wheat_grain": "commodity_prices_wheat_seed_dollar_per_square_meter", + "winter_wheat_silage": "commodity_prices_wheat_seed_dollar_per_square_meter", + "winter_wheat_baleage": "commodity_prices_wheat_seed_dollar_per_square_meter", + "winter_wheat_hay": "commodity_prices_wheat_seed_dollar_per_square_meter", + "triticale_grain": "commodity_prices_wheat_seed_dollar_per_square_meter", + "triticale_silage": "commodity_prices_wheat_seed_dollar_per_square_meter", + "triticale_baleage": "commodity_prices_wheat_seed_dollar_per_square_meter", + "triticale_hay": "commodity_prices_wheat_seed_dollar_per_square_meter", + "cereal_rye_grain": "commodity_prices_wheat_seed_dollar_per_square_meter", + "cereal_rye_silage": "commodity_prices_wheat_seed_dollar_per_square_meter", + "cereal_rye_baleage": "commodity_prices_wheat_seed_dollar_per_square_meter", + "cereal_rye_hay": "commodity_prices_wheat_seed_dollar_per_square_meter", + } + + _HA_TO_M2: float = 10_000.0 + + # Harvest operations that terminate a crop's life in the field. + _FINAL_HARVEST_OPS: frozenset[str] = frozenset({"harvest_kill", "kill_only"}) + + @staticmethod + def _expand_years_with_pattern(years: List[int], skip: int, repeat: int) -> List[int]: + """Expand a year list by repeating its spacing pattern, mirroring Schedule.repeat_pattern.""" + if not years or repeat <= 0: + return list(years) + differences = [skip + 1] + for i in range(1, len(years)): + differences.append(years[i] - years[i - 1]) + full = list(years) + diff_idx = 0 + for _ in range(repeat * len(years)): + full.append(full[-1] + differences[diff_idx]) + diff_idx = (diff_idx + 1) % len(years) + return full + + @staticmethod + def _elongate(lst: List[Any], target_len: int) -> List[Any]: + """Repeat a single-element list to match target_len, otherwise return as-is.""" + if len(lst) == 1 and target_len > 1: + return lst * target_len + return list(lst) + + def _growing_periods( + self, schedule: Dict[str, Any] + ) -> List[Tuple[datetime, datetime]]: + """Return (planting_date, kill_date) pairs for one crop schedule entry. + + Pairs each planting event with its corresponding final harvest + (``harvest_kill`` or ``kill_only``). Intermediate ``harvest_only`` + operations are ignored. Pattern expansion (``pattern_repeat``, + ``planting_skip``, ``harvesting_skip``) is applied before pairing. + """ + pattern_repeat = int(schedule.get("pattern_repeat") or 0) + planting_skip = int(schedule.get("planting_skip") or 0) + harvesting_skip = int(schedule.get("harvesting_skip") or 0) + + raw_p_years: List[int] = list(schedule.get("planting_years") or []) + raw_p_days: List[int] = list(schedule.get("planting_days") or []) + raw_h_years: List[int] = list(schedule.get("harvest_years") or []) + raw_h_days: List[int] = list(schedule.get("harvest_days") or []) + raw_h_ops: List[str] = list(schedule.get("harvest_operations") or []) + + if not raw_p_years or not raw_h_years: + return [] + + p_years = self._expand_years_with_pattern(raw_p_years, planting_skip, pattern_repeat) + p_days = self._elongate(raw_p_days * (pattern_repeat + 1), len(p_years)) + h_years = self._expand_years_with_pattern(raw_h_years, harvesting_skip, pattern_repeat) + h_days = self._elongate(raw_h_days * (pattern_repeat + 1), len(h_years)) + h_ops = self._elongate(raw_h_ops * (pattern_repeat + 1), len(h_years)) + + events: List[Tuple[datetime, str]] = [] + for y, d in zip(p_years, p_days): + events.append((datetime.strptime(f"{y}:{d}", "%Y:%j"), "plant")) + for y, d, op in zip(h_years, h_days, h_ops): + events.append((datetime.strptime(f"{y}:{d}", "%Y:%j"), op)) + events.sort(key=lambda e: e[0]) + + periods: List[Tuple[datetime, datetime]] = [] + plant_date: datetime | None = None + for date, op in events: + if op == "plant": + plant_date = date + elif op in self._FINAL_HARVEST_OPS and plant_date is not None: + periods.append((plant_date, date)) + plant_date = None + return periods + + def _preprocess_seed_costs(self) -> Dict[str, List[float]]: + """Build a daily time-series of responsible field area (m²) per seed key. + + For each field, each crop's planting-to-kill periods are located within + the simulation window. The field's area in m² is spread evenly over + each growing period (``field_size_m² / period_duration``), then + accumulated into a per-simulation-day array keyed by seed commodity. + + Returns + ------- + dict[str, list[float]] + Keys are seed commodity price keys; values are lists of length + ``total_sim_days`` where each element is the total m² for that + seed on that simulation day. + """ + info_map = {"class": self.__class__.__name__, "function": "_preprocess_seed_costs"} + + try: + start_date = datetime.strptime( + str(self.im.get_data("config.start_date")), "%Y:%j" + ) + end_date = datetime.strptime( + str(self.im.get_data("config.end_date")), "%Y:%j" + ) + except Exception: + self.om.add_warning( + "MissingConfigDates", + "Could not parse simulation start/end dates for seed cost preprocessing", + info_map, + ) + return {} + + total_sim_days: int = (end_date - start_date).days + 1 + + try: + field_keys = self.im.get_data_keys_by_properties("field_properties") + except Exception: + self.om.add_warning( + "MissingFieldData", + "Could not retrieve field keys for seed cost preprocessing", + info_map, + ) + return {} + + daily_area_by_seed: Dict[str, List[float]] = {} + + for field_key in field_keys: + field_data = self.im.get_data(field_key) + if not isinstance(field_data, dict): + continue + crop_spec = field_data.get("crop_specification") + field_size_ha = field_data.get("field_size") + if crop_spec is None or field_size_ha is None: + continue + try: + field_size_m2 = float(field_size_ha) * self._HA_TO_M2 + except (TypeError, ValueError): + continue + + crop_schedules = self.im.get_data(f"{crop_spec}.crop_schedules") + if not isinstance(crop_schedules, list) or not crop_schedules: + self.om.add_warning( + "MissingCropSchedule", + f"No crop schedules found for '{crop_spec}' in field '{field_key}'", + info_map, + ) + continue + + for schedule in crop_schedules: + if not isinstance(schedule, dict): + continue + crop_species = schedule.get("crop_species") + if not isinstance(crop_species, str): + continue + seed_key = self._CROP_TO_SEED_KEY.get(crop_species) + if seed_key is None: + continue + + if seed_key not in daily_area_by_seed: + daily_area_by_seed[seed_key] = [0.0] * total_sim_days + + arr = daily_area_by_seed[seed_key] + for plant_date, kill_date in self._growing_periods(schedule): + plant_idx = (plant_date - start_date).days + kill_idx = (kill_date - start_date).days + + # Clip to simulation window. + clipped_start = max(plant_idx, 0) + clipped_end = min(kill_idx, total_sim_days) + if clipped_start >= clipped_end: + continue + + duration = kill_idx - plant_idx + daily_value = field_size_m2 / duration + for i in range(clipped_start, clipped_end): + arr[i] += daily_value + + return daily_area_by_seed + + def _process_seed_costs_item(self) -> Dict[str, Any]: + """Build the full preprocessing result entry for the Seeds costs line item.""" + + info_map = {"class": self.__class__.__name__, "function": "_process_seed_costs_item"} + + daily_area_by_seed = self._preprocess_seed_costs() + + # biophysical_values: concatenation of all per-seed daily arrays. + biophysical_values: List[float] = [] + price_values: List[float] = [] + total_seed_cost = 0.0 + + for seed_key, daily_area in daily_area_by_seed.items(): + price_data = self._get_data_with_handling(seed_key, info_map) + if price_data is None: + self.om.add_warning( + "MissingEconomicsFile", + f"Seed commodity pricing '{seed_key}' not found in InputManager", + info_map, + ) + continue + + extracted_prices = self._extract_price_values(price_data) + if not extracted_prices: + continue + + avg_price = self._aggregate(extracted_prices, "average") + if avg_price is None: + continue + + biophysical_values.extend(daily_area) + price_values.extend(extracted_prices) + total_seed_cost += sum(daily_area) * avg_price + + if not daily_area_by_seed: + self.om.add_warning( + "MissingBiophysicalData", + "No field data found for seed cost preprocessing", + info_map, + ) + + bio_total = sum(biophysical_values) if biophysical_values else None + return { + "biophysical_values": biophysical_values, + "biophysical_aggregate": bio_total, + "biophysical_values_by_scenario": {"baseline": biophysical_values}, + "biophysical_aggregate_by_scenario": {"baseline": bio_total}, + "price_data": {}, + "price_values": price_values, + "price_aggregate": self._aggregate(price_values, "average") if price_values else None, + "line_item_values_by_scenario": {"baseline": total_seed_cost}, + "flow_type": "cost", + } + def _aggregate(self, values: List[float], desc: str) -> float | None: """Aggregate values according to a textual description.""" if not values: @@ -665,6 +912,10 @@ def preprocess(self) -> Dict[str, Dict[str, Dict[str, Dict[str, Any]]]]: section_data = results.setdefault(item.section, {}) category_data = section_data.setdefault(item.category, {}) + # if item.section == "Soil_and_crop" and item.name == "Seeds costs": + # category_data[item.name] = self._process_seed_costs_item() + # continue + values_by_scenario = self._fetch_values_by_scenario(item.biophysical_simulation) wildcard_values = self._collect_biophysical_wildcards(item.biophysical_simulation) input_values, input_match_values = self._fetch_input_values( @@ -765,6 +1016,7 @@ def preprocess(self) -> Dict[str, Dict[str, Dict[str, Dict[str, Any]]]]: data=results, properties_blob_key="economic_preprocessing_properties", eager_termination=False, + input_path=Path() ) self.om.add_log( "Economic preprocessing", diff --git a/tests/test_EEE/test_economics_preprocessing.py b/tests/test_EEE/test_economics_preprocessing.py index bc7fbde101..70edea8206 100644 --- a/tests/test_EEE/test_economics_preprocessing.py +++ b/tests/test_EEE/test_economics_preprocessing.py @@ -28,21 +28,29 @@ def add_log(self, title, message, info): class DummyInputManager: - def __init__(self, data): + def __init__(self, data, field_keys=None): self._data = { - "config.start_date": "2020:01:01", - "config.end_date": "2020:12:31", + "config.start_date": "2020:1", + "config.end_date": "2020:365", "config.FIPS_county_code": 1001, "_default_values": {"commodity": [], "2020": []}, "_default_fallback_values": {"commodity": [], "2020": []}, **data, } + self._field_keys = field_keys or [] self.added_runtime = [] def get_data(self, key): return self._data.get(key) - def add_runtime_variable_to_pool(self, variable_name, data, properties_blob_key, eager_termination=False): + def get_data_keys_by_properties(self, properties_key): + if properties_key == "field_properties": + return self._field_keys + return [] + + def add_runtime_variable_to_pool( + self, variable_name, data, properties_blob_key, eager_termination=False, input_path=None + ): self.added_runtime.append( { "variable_name": variable_name, @@ -528,3 +536,201 @@ def test_preprocess_expands_input_wildcard_with_value_map(monkeypatch: pytest.Mo {"X": "A", "Y": "B", "Z": "C"}, ) assert values == [5.0, 6.0, 7.0] + + +def _seed_cost_map(): + """Minimal ECONOMIC_MAP for seed cost tests.""" + return { + "Soil_and_crop": { + "Costs": { + "Seeds costs": { + "biophysical_simulation": ["field.crop_specification", "field.field_size"], + "economics_files": ["commodity_prices_corn_seed_dollar_per_square_meter"], + } + } + } + } + + +def _corn_schedule(plant_day: int, kill_day: int, year: int = 2020) -> dict: + """Build a minimal corn_silage crop schedule dict for tests.""" + return { + "crop_species": "corn_silage", + "planting_years": [year], + "planting_days": [plant_day], + "harvest_years": [year], + "harvest_days": [kill_day], + "harvest_operations": ["harvest_kill"], + "pattern_repeat": 0, + "planting_skip": 0, + "harvesting_skip": 0, + } + + +def test_preprocess_seed_costs_daily_array_shape_and_values(monkeypatch: pytest.MonkeyPatch) -> None: + """Daily area array has length == sim days with correct values inside the growing period. + + Sim: 2020:1 → 2020:365 (365 days, indices 0-364) + field_a: 1 ha corn_silage, planted day 100, killed day 200. + plant_idx = 99, kill_idx = 199, duration = 100 + daily_value = 10,000 / 100 = 100.0 + Array is 100.0 for indices 99-198, 0.0 elsewhere. + """ + dummy_im = DummyInputManager( + data={ + "field_a": {"crop_specification": "RotA", "field_size": 1.0}, + "RotA.crop_schedules": [_corn_schedule(100, 200)], + "commodity_prices_corn_seed_dollar_per_square_meter": { + "corn_seed": {"fips": [1001], "2020": [0.01]} + }, + }, + field_keys=["field_a"], + ) + dummy_om = DummyOutputManager({}) + monkeypatch.setattr(preprocessing, "InputManager", lambda: dummy_im) + monkeypatch.setattr(preprocessing, "OutputManager", lambda: dummy_om) + + preprocessor = preprocessing.EconomicPreprocessor() + daily = preprocessor._preprocess_seed_costs() + + seed_key = "commodity_prices_corn_seed_dollar_per_square_meter" + assert seed_key in daily + arr = daily[seed_key] + assert len(arr) == 365 + assert arr[98] == pytest.approx(0.0) # day 99 (index 98) — before planting + assert arr[99] == pytest.approx(100.0) # planting day (index 99) + assert arr[198] == pytest.approx(100.0) # last day of growth (index 198) + assert arr[199] == pytest.approx(0.0) # harvest day — not included + assert sum(arr) == pytest.approx(10_000.0) # 100 days × 100 m²/day + + +def test_preprocess_seed_costs_computes_total_cost(monkeypatch: pytest.MonkeyPatch) -> None: + """Total seed cost = sum(daily_area) × avg_price across all fields and seed types. + + Sim: 2020:1 → 2020:365 + field_a: 1 ha corn_silage, day 100→200: sum = 10,000 m² + field_b: 2 ha corn_grain, day 50→150: sum = 20,000 m² + 2 ha alfalfa_silage (no seed key) — skipped + corn_seed price = $0.01/m² → total = 30,000 × 0.01 = $300 + """ + dummy_im = DummyInputManager( + data={ + "field_a": {"crop_specification": "RotA", "field_size": 1.0}, + "field_b": {"crop_specification": "RotB", "field_size": 2.0}, + "RotA.crop_schedules": [_corn_schedule(100, 200)], + "RotB.crop_schedules": [ + { + "crop_species": "corn_grain", + "planting_years": [2020], + "planting_days": [50], + "harvest_years": [2020], + "harvest_days": [150], + "harvest_operations": ["harvest_kill"], + "pattern_repeat": 0, + "planting_skip": 0, + "harvesting_skip": 0, + }, + { + "crop_species": "alfalfa_silage", + "planting_years": [2020], + "planting_days": [155], + "harvest_years": [2020], + "harvest_days": [300], + "harvest_operations": ["harvest_kill"], + "pattern_repeat": 0, + "planting_skip": 0, + "harvesting_skip": 0, + }, + ], + "commodity_prices_corn_seed_dollar_per_square_meter": { + "corn_seed": {"fips": [1001], "2020": [0.01]} + }, + }, + field_keys=["field_a", "field_b"], + ) + dummy_om = DummyOutputManager({}) + monkeypatch.setattr(preprocessing, "InputManager", lambda: dummy_im) + monkeypatch.setattr(preprocessing, "OutputManager", lambda: dummy_om) + monkeypatch.setattr(preprocessing, "ECONOMIC_MAP", _seed_cost_map()) + + preprocessor = preprocessing.EconomicPreprocessor() + results = preprocessor.preprocess() + + item = results["Soil_and_crop"]["Costs"]["Seeds costs"] + assert item["flow_type"] == "cost" + assert item["line_item_values_by_scenario"]["baseline"] == pytest.approx(300.0) + + +def test_preprocess_seed_costs_warns_on_missing_field_data(monkeypatch: pytest.MonkeyPatch) -> None: + """Seed costs handler emits warning when no field keys are available.""" + + dummy_im = DummyInputManager(data={}, field_keys=[]) + dummy_om = DummyOutputManager({}) + + monkeypatch.setattr(preprocessing, "InputManager", lambda: dummy_im) + monkeypatch.setattr(preprocessing, "OutputManager", lambda: dummy_om) + monkeypatch.setattr(preprocessing, "ECONOMIC_MAP", _seed_cost_map()) + + preprocessor = preprocessing.EconomicPreprocessor() + results = preprocessor.preprocess() + + item = results["Soil_and_crop"]["Costs"]["Seeds costs"] + assert item["line_item_values_by_scenario"]["baseline"] == pytest.approx(0.0) + warning_codes = [code for code, _, _ in dummy_om.warnings] + assert "MissingBiophysicalData" in warning_codes + + +def test_preprocess_seed_costs_clips_period_to_simulation_window(monkeypatch: pytest.MonkeyPatch) -> None: + """Growing periods that extend beyond the simulation window are clipped. + + Sim: 2020:1 → 2020:365 (365 days, indices 0-364) + field: 1 ha corn_silage, planted day 300, killed day 100 of 2021. + plant_idx = 299, kill_idx = (2021_day100 - 2020_day1).days = 464 + Clipped range: [299, 365) → 66 days active + duration (unclipped) = 464 - 299 = 165 + daily_value = 10,000 / 165 ≈ 60.6 + sum(arr) ≈ 66 × 60.6 ≈ 4,000 (= 10,000 × 66/165) + """ + dummy_im = DummyInputManager( + data={ + "field_a": {"crop_specification": "RotA", "field_size": 1.0}, + "RotA.crop_schedules": [ + { + "crop_species": "corn_silage", + "planting_years": [2020], + "planting_days": [300], + "harvest_years": [2021], + "harvest_days": [100], + "harvest_operations": ["harvest_kill"], + "pattern_repeat": 0, + "planting_skip": 0, + "harvesting_skip": 0, + } + ], + }, + field_keys=["field_a"], + ) + dummy_om = DummyOutputManager({}) + monkeypatch.setattr(preprocessing, "InputManager", lambda: dummy_im) + monkeypatch.setattr(preprocessing, "OutputManager", lambda: dummy_om) + + preprocessor = preprocessing.EconomicPreprocessor() + daily = preprocessor._preprocess_seed_costs() + + seed_key = "commodity_prices_corn_seed_dollar_per_square_meter" + arr = daily[seed_key] + assert len(arr) == 365 + assert arr[298] == pytest.approx(0.0) # before planting + + from datetime import datetime + + plant_date = datetime.strptime("2020:300", "%Y:%j") + kill_date = datetime.strptime("2021:100", "%Y:%j") + start_date = datetime.strptime("2020:1", "%Y:%j") + duration = (kill_date - plant_date).days + expected_daily = 10_000.0 / duration + expected_sum = 10_000.0 * 66 / duration + + assert arr[299] == pytest.approx(expected_daily) + assert arr[364] == pytest.approx(expected_daily) + assert sum(arr) == pytest.approx(expected_sum) From 3900c3729f16bbcbb21d7822f178fcc3c365eaff Mon Sep 17 00:00:00 2001 From: Allister Liu Date: Thu, 25 Jun 2026 10:02:32 -0400 Subject: [PATCH 2/9] update --- RUFAS/EEE/economics/preprocessing.py | 12 ++++++------ tests/test_EEE/test_economics_preprocessing.py | 8 ++------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/RUFAS/EEE/economics/preprocessing.py b/RUFAS/EEE/economics/preprocessing.py index b13aeea8aa..3f26e19cbd 100644 --- a/RUFAS/EEE/economics/preprocessing.py +++ b/RUFAS/EEE/economics/preprocessing.py @@ -839,8 +839,8 @@ def _process_seed_costs_item(self) -> Dict[str, Any]: total_seed_cost = 0.0 for seed_key, daily_area in daily_area_by_seed.items(): - price_data = self._get_data_with_handling(seed_key, info_map) - if price_data is None: + raw_price = self._get_data_with_handling(seed_key, info_map) + if raw_price is None: self.om.add_warning( "MissingEconomicsFile", f"Seed commodity pricing '{seed_key}' not found in InputManager", @@ -848,7 +848,7 @@ def _process_seed_costs_item(self) -> Dict[str, Any]: ) continue - extracted_prices = self._extract_price_values(price_data) + extracted_prices = self._extract_price_values({seed_key: raw_price}) if not extracted_prices: continue @@ -912,9 +912,9 @@ def preprocess(self) -> Dict[str, Dict[str, Dict[str, Dict[str, Any]]]]: section_data = results.setdefault(item.section, {}) category_data = section_data.setdefault(item.category, {}) - # if item.section == "Soil_and_crop" and item.name == "Seeds costs": - # category_data[item.name] = self._process_seed_costs_item() - # continue + if item.section == "Soil_and_crop" and item.name == "Seeds costs": + category_data[item.name] = self._process_seed_costs_item() + continue values_by_scenario = self._fetch_values_by_scenario(item.biophysical_simulation) wildcard_values = self._collect_biophysical_wildcards(item.biophysical_simulation) diff --git a/tests/test_EEE/test_economics_preprocessing.py b/tests/test_EEE/test_economics_preprocessing.py index 70edea8206..f8810abd83 100644 --- a/tests/test_EEE/test_economics_preprocessing.py +++ b/tests/test_EEE/test_economics_preprocessing.py @@ -580,9 +580,7 @@ def test_preprocess_seed_costs_daily_array_shape_and_values(monkeypatch: pytest. data={ "field_a": {"crop_specification": "RotA", "field_size": 1.0}, "RotA.crop_schedules": [_corn_schedule(100, 200)], - "commodity_prices_corn_seed_dollar_per_square_meter": { - "corn_seed": {"fips": [1001], "2020": [0.01]} - }, + "commodity_prices_corn_seed_dollar_per_square_meter": {"fips": [1001], "2020": [0.01]}, }, field_keys=["field_a"], ) @@ -642,9 +640,7 @@ def test_preprocess_seed_costs_computes_total_cost(monkeypatch: pytest.MonkeyPat "harvesting_skip": 0, }, ], - "commodity_prices_corn_seed_dollar_per_square_meter": { - "corn_seed": {"fips": [1001], "2020": [0.01]} - }, + "commodity_prices_corn_seed_dollar_per_square_meter": {"fips": [1001], "2020": [0.01]}, }, field_keys=["field_a", "field_b"], ) From 16ab32fd726f4662d8ada94d2a93495cdc3a9fb8 Mon Sep 17 00:00:00 2001 From: Allister Liu Date: Thu, 25 Jun 2026 10:42:01 -0400 Subject: [PATCH 3/9] Update preprocessing.py --- RUFAS/EEE/economics/preprocessing.py | 74 ++++++++++++++++++++++------ 1 file changed, 60 insertions(+), 14 deletions(-) diff --git a/RUFAS/EEE/economics/preprocessing.py b/RUFAS/EEE/economics/preprocessing.py index 3f26e19cbd..0b294083fb 100644 --- a/RUFAS/EEE/economics/preprocessing.py +++ b/RUFAS/EEE/economics/preprocessing.py @@ -17,7 +17,7 @@ import math import re from dataclasses import dataclass -from datetime import datetime +from datetime import datetime, timedelta from pathlib import Path from typing import Any, Dict, Iterable, List, Set, Tuple @@ -727,7 +727,7 @@ def _growing_periods( plant_date = None return periods - def _preprocess_seed_costs(self) -> Dict[str, List[float]]: + def _preprocess_seed_costs(self) -> dict[str, List[float]]: """Build a daily time-series of responsible field area (m²) per seed key. For each field, each crop's planting-to-kill periods are located within @@ -771,7 +771,7 @@ def _preprocess_seed_costs(self) -> Dict[str, List[float]]: ) return {} - daily_area_by_seed: Dict[str, List[float]] = {} + daily_area_by_seed: dict[str, List[float]] = {} for field_key in field_keys: field_data = self.im.get_data(field_key) @@ -826,6 +826,49 @@ def _preprocess_seed_costs(self) -> Dict[str, List[float]]: return daily_area_by_seed + def _extract_daily_seed_price(self, price_data: Any) -> list[float]: + info_map = {"class": self.__class__.__name__, "function": self._extract_price_values.__name__} + config_data = self.im.get_data("config") + start_date = datetime.strptime( + str(config_data["start_date"]), "%Y:%j" + ) + end_date = datetime.strptime( + str(config_data["end_date"]), "%Y:%j" + ) + start_year: int = start_date.year + end_year: int = end_date.year + fips_code: int = int(config_data["start_date"]) + days_count = (end_date - start_date).days + date_generator = (start_date + timedelta(days=i) for i in range(days_count)) + + daily_seed_price: list[float] = [] + for key, value in price_data.items(): + if not isinstance(value, dict) or "fips" not in value or not isinstance(value["fips"], list): + self.om.add_warning( + "MissingPriceData", + f"Price data missing for key: {key}, FIPS: '{fips_code}' is not in expected format." + "Using fallback price.", + info_map, + ) + daily_seed_price.extend(self._get_fallback_price(start_year, end_year, key)) + continue + fips_idx = value["fips"].index(fips_code) + for date in date_generator: + year = date.year + try: + price = value[f"{year}"][fips_idx] + daily_seed_price.append(price) + except (KeyError, IndexError): + self.om.add_warning( + "MissingPriceData", + f"Price data missing for year '{year}' and FIPS '{fips_code}' in '{key}'." + "Using fallback price.", + info_map, + ) + daily_seed_price.extend(self._get_fallback_price(start_year, end_year, key)) + continue + return daily_seed_price + def _process_seed_costs_item(self) -> Dict[str, Any]: """Build the full preprocessing result entry for the Seeds costs line item.""" @@ -834,8 +877,8 @@ def _process_seed_costs_item(self) -> Dict[str, Any]: daily_area_by_seed = self._preprocess_seed_costs() # biophysical_values: concatenation of all per-seed daily arrays. - biophysical_values: List[float] = [] - price_values: List[float] = [] + biophysical_values: dict[str, list[float]] = {} + price_values: dict[str, list[float]] = {} total_seed_cost = 0.0 for seed_key, daily_area in daily_area_by_seed.items(): @@ -848,17 +891,20 @@ def _process_seed_costs_item(self) -> Dict[str, Any]: ) continue - extracted_prices = self._extract_price_values({seed_key: raw_price}) + extracted_prices = self._extract_daily_seed_price({seed_key: raw_price}) if not extracted_prices: continue - avg_price = self._aggregate(extracted_prices, "average") - if avg_price is None: - continue + # avg_price = self._aggregate(extracted_prices, "average") + # if avg_price is None: + # continue - biophysical_values.extend(daily_area) - price_values.extend(extracted_prices) - total_seed_cost += sum(daily_area) * avg_price + daily_price_per_area = [ + seed_cost * area_m2 for seed_cost, area_m2 in zip(extracted_prices, daily_area_by_seed[seed_key]) + ] + biophysical_values[seed_key] = extracted_prices # is that right? + price_values[seed_key] = daily_price_per_area # is that right? + total_seed_cost += sum(daily_price_per_area) # is that right? if not daily_area_by_seed: self.om.add_warning( @@ -867,7 +913,7 @@ def _process_seed_costs_item(self) -> Dict[str, Any]: info_map, ) - bio_total = sum(biophysical_values) if biophysical_values else None + bio_total = 0.0 # what to put here return { "biophysical_values": biophysical_values, "biophysical_aggregate": bio_total, @@ -875,7 +921,7 @@ def _process_seed_costs_item(self) -> Dict[str, Any]: "biophysical_aggregate_by_scenario": {"baseline": bio_total}, "price_data": {}, "price_values": price_values, - "price_aggregate": self._aggregate(price_values, "average") if price_values else None, + "price_aggregate": 0.0, # what to put here "line_item_values_by_scenario": {"baseline": total_seed_cost}, "flow_type": "cost", } From 152a5033fc6833be1cce8de3f5044a3963dc8941 Mon Sep 17 00:00:00 2001 From: Allister Liu Date: Thu, 25 Jun 2026 11:17:52 -0400 Subject: [PATCH 4/9] bug fixes --- RUFAS/EEE/economics/preprocessing.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/RUFAS/EEE/economics/preprocessing.py b/RUFAS/EEE/economics/preprocessing.py index 0b294083fb..3cc058d607 100644 --- a/RUFAS/EEE/economics/preprocessing.py +++ b/RUFAS/EEE/economics/preprocessing.py @@ -809,7 +809,8 @@ def _preprocess_seed_costs(self) -> dict[str, List[float]]: daily_area_by_seed[seed_key] = [0.0] * total_sim_days arr = daily_area_by_seed[seed_key] - for plant_date, kill_date in self._growing_periods(schedule): + growing_periods = self._growing_periods(schedule) + for plant_date, kill_date in growing_periods: plant_idx = (plant_date - start_date).days kill_idx = (kill_date - start_date).days @@ -837,7 +838,7 @@ def _extract_daily_seed_price(self, price_data: Any) -> list[float]: ) start_year: int = start_date.year end_year: int = end_date.year - fips_code: int = int(config_data["start_date"]) + fips_code: int = int(config_data["FIPS_county_code"]) days_count = (end_date - start_date).days date_generator = (start_date + timedelta(days=i) for i in range(days_count)) @@ -900,7 +901,7 @@ def _process_seed_costs_item(self) -> Dict[str, Any]: # continue daily_price_per_area = [ - seed_cost * area_m2 for seed_cost, area_m2 in zip(extracted_prices, daily_area_by_seed[seed_key]) + seed_cost * area_m2 for seed_cost, area_m2 in zip(extracted_prices, daily_area) ] biophysical_values[seed_key] = extracted_prices # is that right? price_values[seed_key] = daily_price_per_area # is that right? From 9464d9a6d3832d8969cacfa33740c41536576fc5 Mon Sep 17 00:00:00 2001 From: Allister Liu Date: Fri, 26 Jun 2026 16:10:03 -0400 Subject: [PATCH 5/9] update reporting per Braden's feedback --- RUFAS/EEE/economics/preprocessing.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/RUFAS/EEE/economics/preprocessing.py b/RUFAS/EEE/economics/preprocessing.py index 3cc058d607..49e45b02e0 100644 --- a/RUFAS/EEE/economics/preprocessing.py +++ b/RUFAS/EEE/economics/preprocessing.py @@ -877,9 +877,11 @@ def _process_seed_costs_item(self) -> Dict[str, Any]: daily_area_by_seed = self._preprocess_seed_costs() - # biophysical_values: concatenation of all per-seed daily arrays. biophysical_values: dict[str, list[float]] = {} + bio_total: dict[str, float] = {} + price_data: dict[str, list[float]] = {} price_values: dict[str, list[float]] = {} + price_aggregate: dict[str,float] = {} total_seed_cost = 0.0 for seed_key, daily_area in daily_area_by_seed.items(): @@ -896,16 +898,15 @@ def _process_seed_costs_item(self) -> Dict[str, Any]: if not extracted_prices: continue - # avg_price = self._aggregate(extracted_prices, "average") - # if avg_price is None: - # continue - daily_price_per_area = [ seed_cost * area_m2 for seed_cost, area_m2 in zip(extracted_prices, daily_area) ] - biophysical_values[seed_key] = extracted_prices # is that right? - price_values[seed_key] = daily_price_per_area # is that right? - total_seed_cost += sum(daily_price_per_area) # is that right? + biophysical_values[seed_key] = daily_area_by_seed[seed_key] + price_values[seed_key] = extracted_prices + bio_total[seed_key] = sum(biophysical_values[seed_key]) + price_data[seed_key] = raw_price + price_aggregate[seed_key] = self._aggregate(extracted_prices, "average") + total_seed_cost += sum(daily_price_per_area) if not daily_area_by_seed: self.om.add_warning( @@ -914,15 +915,14 @@ def _process_seed_costs_item(self) -> Dict[str, Any]: info_map, ) - bio_total = 0.0 # what to put here return { "biophysical_values": biophysical_values, "biophysical_aggregate": bio_total, "biophysical_values_by_scenario": {"baseline": biophysical_values}, "biophysical_aggregate_by_scenario": {"baseline": bio_total}, - "price_data": {}, + "price_data": price_data, "price_values": price_values, - "price_aggregate": 0.0, # what to put here + "price_aggregate": price_aggregate, "line_item_values_by_scenario": {"baseline": total_seed_cost}, "flow_type": "cost", } From fd226ed71eaa1a8ac4e59d0c89e70bb50ea0f859 Mon Sep 17 00:00:00 2001 From: Allister Liu Date: Mon, 13 Jul 2026 00:10:26 -0400 Subject: [PATCH 6/9] cleanup --- RUFAS/EEE/economics/preprocessing.py | 121 +++++++++++++-------------- tests/test_EEE/test_economics.py | 61 +++++--------- 2 files changed, 79 insertions(+), 103 deletions(-) diff --git a/RUFAS/EEE/economics/preprocessing.py b/RUFAS/EEE/economics/preprocessing.py index 49e45b02e0..cc31ab576b 100644 --- a/RUFAS/EEE/economics/preprocessing.py +++ b/RUFAS/EEE/economics/preprocessing.py @@ -21,9 +21,10 @@ from pathlib import Path from typing import Any, Dict, Iterable, List, Set, Tuple +from RUFAS.biophysical.field.manager.schedule import Schedule from RUFAS.input_manager import InputManager from RUFAS.output_manager import OutputManager -from RUFAS.util import Aggregator +from RUFAS.util import Aggregator, Utility from RUFAS.EEE.economics.mapping import ECONOMIC_MAP from RUFAS.EEE.economics.fallback_values import ( BIOPHYSICAL_FALLBACKS, @@ -659,28 +660,6 @@ def _fetch_prices_with_exact_matches( # Harvest operations that terminate a crop's life in the field. _FINAL_HARVEST_OPS: frozenset[str] = frozenset({"harvest_kill", "kill_only"}) - @staticmethod - def _expand_years_with_pattern(years: List[int], skip: int, repeat: int) -> List[int]: - """Expand a year list by repeating its spacing pattern, mirroring Schedule.repeat_pattern.""" - if not years or repeat <= 0: - return list(years) - differences = [skip + 1] - for i in range(1, len(years)): - differences.append(years[i] - years[i - 1]) - full = list(years) - diff_idx = 0 - for _ in range(repeat * len(years)): - full.append(full[-1] + differences[diff_idx]) - diff_idx = (diff_idx + 1) % len(years) - return full - - @staticmethod - def _elongate(lst: List[Any], target_len: int) -> List[Any]: - """Repeat a single-element list to match target_len, otherwise return as-is.""" - if len(lst) == 1 and target_len > 1: - return lst * target_len - return list(lst) - def _growing_periods( self, schedule: Dict[str, Any] ) -> List[Tuple[datetime, datetime]]: @@ -695,20 +674,20 @@ def _growing_periods( planting_skip = int(schedule.get("planting_skip") or 0) harvesting_skip = int(schedule.get("harvesting_skip") or 0) - raw_p_years: List[int] = list(schedule.get("planting_years") or []) - raw_p_days: List[int] = list(schedule.get("planting_days") or []) - raw_h_years: List[int] = list(schedule.get("harvest_years") or []) - raw_h_days: List[int] = list(schedule.get("harvest_days") or []) - raw_h_ops: List[str] = list(schedule.get("harvest_operations") or []) + raw_p_years: list[int] = list(schedule.get("planting_years") or []) + raw_p_days: list[int] = list(schedule.get("planting_days") or []) + raw_h_years: list[int] = list(schedule.get("harvest_years") or []) + raw_h_days: list[int] = list(schedule.get("harvest_days") or []) + raw_h_ops: list[str] = list(schedule.get("harvest_operations") or []) if not raw_p_years or not raw_h_years: return [] - p_years = self._expand_years_with_pattern(raw_p_years, planting_skip, pattern_repeat) - p_days = self._elongate(raw_p_days * (pattern_repeat + 1), len(p_years)) - h_years = self._expand_years_with_pattern(raw_h_years, harvesting_skip, pattern_repeat) - h_days = self._elongate(raw_h_days * (pattern_repeat + 1), len(h_years)) - h_ops = self._elongate(raw_h_ops * (pattern_repeat + 1), len(h_years)) + p_years = Schedule.repeat_pattern(raw_p_years, planting_skip, pattern_repeat) + p_days = Utility.elongate_list(raw_p_days * (pattern_repeat + 1), len(p_years)) + h_years = Schedule.repeat_pattern(raw_h_years, harvesting_skip, pattern_repeat) + h_days = Utility.elongate_list(raw_h_days * (pattern_repeat + 1), len(h_years)) + h_ops = Utility.elongate_list(raw_h_ops * (pattern_repeat + 1), len(h_years)) events: List[Tuple[datetime, str]] = [] for y, d in zip(p_years, p_days): @@ -828,22 +807,36 @@ def _preprocess_seed_costs(self) -> dict[str, List[float]]: return daily_area_by_seed def _extract_daily_seed_price(self, price_data: Any) -> list[float]: - info_map = {"class": self.__class__.__name__, "function": self._extract_price_values.__name__} - config_data = self.im.get_data("config") - start_date = datetime.strptime( - str(config_data["start_date"]), "%Y:%j" - ) - end_date = datetime.strptime( - str(config_data["end_date"]), "%Y:%j" - ) + """Extract a per-simulation-day price series from seed pricing payloads. + + Yearly prices are resolved per commodity key (with fallback prices for + malformed payloads or missing years), then expanded so each simulation + day carries its year's price. + + Parameters + ---------- + price_data : Any + Mapping of commodity price keys to pricing payloads keyed by year + and FIPS code. + + Returns + ------- + list[float] + One price per simulation day, matching the length of the daily + area series produced by ``_preprocess_seed_costs``. + """ + info_map = {"class": self.__class__.__name__, "function": self._extract_daily_seed_price.__name__} + start_date = datetime.strptime(str(self.im.get_data("config.start_date")), "%Y:%j") + end_date = datetime.strptime(str(self.im.get_data("config.end_date")), "%Y:%j") start_year: int = start_date.year end_year: int = end_date.year - fips_code: int = int(config_data["FIPS_county_code"]) - days_count = (end_date - start_date).days - date_generator = (start_date + timedelta(days=i) for i in range(days_count)) + fips_code: int = int(self.im.get_data("config.FIPS_county_code")) + days_count = (end_date - start_date).days + 1 daily_seed_price: list[float] = [] for key, value in price_data.items(): + fallback_prices: list[float] | None = None + price_by_year: dict[int, float] = {} if not isinstance(value, dict) or "fips" not in value or not isinstance(value["fips"], list): self.om.add_warning( "MissingPriceData", @@ -851,23 +844,25 @@ def _extract_daily_seed_price(self, price_data: Any) -> list[float]: "Using fallback price.", info_map, ) - daily_seed_price.extend(self._get_fallback_price(start_year, end_year, key)) - continue - fips_idx = value["fips"].index(fips_code) - for date in date_generator: - year = date.year - try: - price = value[f"{year}"][fips_idx] - daily_seed_price.append(price) - except (KeyError, IndexError): - self.om.add_warning( - "MissingPriceData", - f"Price data missing for year '{year}' and FIPS '{fips_code}' in '{key}'." - "Using fallback price.", - info_map, - ) - daily_seed_price.extend(self._get_fallback_price(start_year, end_year, key)) - continue + fallback_prices = self._get_fallback_price(start_year, end_year, key) + price_by_year = {start_year + i: price for i, price in enumerate(fallback_prices)} + else: + fips_idx = value["fips"].index(fips_code) + for year in range(start_year, end_year + 1): + try: + price_by_year[year] = value[f"{year}"][fips_idx] + except (KeyError, IndexError): + self.om.add_warning( + "MissingPriceData", + f"Price data missing for year '{year}' and FIPS '{fips_code}' in '{key}'." + "Using fallback price.", + info_map, + ) + if fallback_prices is None: + fallback_prices = self._get_fallback_price(start_year, end_year, key) + price_by_year[year] = fallback_prices[year - start_year] + for i in range(days_count): + daily_seed_price.append(price_by_year[(start_date + timedelta(days=i)).year]) return daily_seed_price def _process_seed_costs_item(self) -> Dict[str, Any]: @@ -881,7 +876,7 @@ def _process_seed_costs_item(self) -> Dict[str, Any]: bio_total: dict[str, float] = {} price_data: dict[str, list[float]] = {} price_values: dict[str, list[float]] = {} - price_aggregate: dict[str,float] = {} + price_aggregate: dict[str, float] = {} total_seed_cost = 0.0 for seed_key, daily_area in daily_area_by_seed.items(): @@ -1063,7 +1058,7 @@ def preprocess(self) -> Dict[str, Dict[str, Dict[str, Dict[str, Any]]]]: data=results, properties_blob_key="economic_preprocessing_properties", eager_termination=False, - input_path=Path() + input_path=Path(), ) self.om.add_log( "Economic preprocessing", diff --git a/tests/test_EEE/test_economics.py b/tests/test_EEE/test_economics.py index 2687aef89d..67bd78f357 100644 --- a/tests/test_EEE/test_economics.py +++ b/tests/test_EEE/test_economics.py @@ -10,26 +10,7 @@ from RUFAS.EEE.economics.digester_costs import ( DigesterCostCalculator, ) -from RUFAS.EEE.economics.equations import ( - construct_timeline, - discount_factor, - annual_capital_spent, - equity_contribution, - loan_principal, - construction_interest, - npv_capital_plus_interest, - annual_loan_payment, - interest_payment, - principal_after_payment, - depreciation_schedule, - net_revenue, - loss_carry_forward, - taxable_income, - income_tax, - annual_cash_income, - present_value, - net_present_value, -) +from RUFAS.EEE.economics.equations import EconomicEquations def test_run_economic_analysis_uses_dcfror_when_capital_present( @@ -309,60 +290,60 @@ def test_dcfror_prepare_costs_applies_digester_cost_curve(mocker: MockerFixture) def test_equation_helpers() -> None: - years = construct_timeline(2, 5) + years = EconomicEquations.construct_timeline(2, 5) assert years.tolist() == [-1, 0, 1, 2, 3, 4, 5] - df = discount_factor(0.1, 2) + df = EconomicEquations.discount_factor(0.1, 2) assert pytest.approx(df) == 1 / (1.1**2) - assert pytest.approx(discount_factor(0.1, -1)) == 1 / 1.1 + assert pytest.approx(EconomicEquations.discount_factor(0.1, -1)) == 1 / 1.1 construction_rates = [0.5, 0.5] - cap = annual_capital_spent(100.0, construction_rates) + cap = EconomicEquations.annual_capital_spent(100.0, construction_rates) assert cap.tolist() == [50.0, 50.0] - equity = equity_contribution(100.0, construction_rates, 0.4) + equity = EconomicEquations.equity_contribution(100.0, construction_rates, 0.4) assert equity.tolist() == [20.0, 20.0] - lp = loan_principal(100.0, construction_rates, 0.6) + lp = EconomicEquations.loan_principal(100.0, construction_rates, 0.6) assert lp.tolist() == [30.0, 30.0] - ci = construction_interest(lp, 0.05) + ci = EconomicEquations.construction_interest(lp, 0.05) assert ci.tolist() == [1.5, 3.0] - npv_ci = npv_capital_plus_interest(cap, ci, 0.1, np.array([-1, 0])) - expected_npv = (cap + ci) * np.array([discount_factor(0.1, -1), discount_factor(0.1, 0)]) + npv_ci = EconomicEquations.npv_capital_plus_interest(cap, ci, 0.1, np.array([-1, 0])) + expected_npv = (cap + ci) * np.array([EconomicEquations.discount_factor(0.1, -1), EconomicEquations.discount_factor(0.1, 0)]) assert np.allclose(npv_ci, expected_npv) - payment = annual_loan_payment(1000.0, 0.05, 5, 0.8) + payment = EconomicEquations.annual_loan_payment(1000.0, 0.05, 5, 0.8) expected_payment = 1000.0 * 0.05 * 0.8 / (1 - (1 + 0.05) ** -5) assert pytest.approx(payment) == expected_payment - int_pay = interest_payment(800.0, 0.05) + int_pay = EconomicEquations.interest_payment(800.0, 0.05) assert pytest.approx(int_pay) == 40.0 - remaining = principal_after_payment(800.0, payment, int_pay) + remaining = EconomicEquations.principal_after_payment(800.0, payment, int_pay) assert pytest.approx(remaining) == 800.0 - payment + int_pay - dep = depreciation_schedule(1000.0, np.array([0.1, 0.2])) + dep = EconomicEquations.depreciation_schedule(1000.0, np.array([0.1, 0.2])) assert dep.tolist() == [100.0, 200.0] - nr = net_revenue(500.0, 200.0, 30.0, 20.0) + nr = EconomicEquations.net_revenue(500.0, 200.0, 30.0, 20.0) assert nr == 250.0 - loss = loss_carry_forward(-50.0) + loss = EconomicEquations.loss_carry_forward(-50.0) assert loss == -50.0 - taxable = taxable_income(nr, loss) + taxable = EconomicEquations.taxable_income(nr, loss) assert taxable == 200.0 - tax = income_tax(taxable, 0.3) + tax = EconomicEquations.income_tax(taxable, 0.3) assert tax == 60.0 - aci = annual_cash_income(500.0, 200.0, 50.0, tax) + aci = EconomicEquations.annual_cash_income(500.0, 200.0, 50.0, tax) assert aci == 190.0 - pv = present_value(aci, discount_factor(0.1, 1)) + pv = EconomicEquations.present_value(aci, EconomicEquations.discount_factor(0.1, 1)) assert pytest.approx(pv) == aci / 1.1 - npv = net_present_value(np.array([pv]), np.array([10.0])) + npv = EconomicEquations.net_present_value(np.array([pv]), np.array([10.0])) assert pytest.approx(npv) == pv - 10.0 From f182bd380cf0827da903ea1f23cae3ee8633bdc3 Mon Sep 17 00:00:00 2001 From: Allister Liu Date: Mon, 13 Jul 2026 10:44:24 -0400 Subject: [PATCH 7/9] update --- RUFAS/EEE/economics/preprocessing.py | 126 +++++++++++------- .../test_EEE/test_economics_preprocessing.py | 36 +++++ 2 files changed, 115 insertions(+), 47 deletions(-) diff --git a/RUFAS/EEE/economics/preprocessing.py b/RUFAS/EEE/economics/preprocessing.py index cc31ab576b..287ebf8fa7 100644 --- a/RUFAS/EEE/economics/preprocessing.py +++ b/RUFAS/EEE/economics/preprocessing.py @@ -14,6 +14,7 @@ from __future__ import annotations +import json import math import re from dataclasses import dataclass @@ -234,16 +235,16 @@ def _fetch_values(self, sim_paths: Iterable[str]) -> List[float]: ) return values - def _fetch_values_by_scenario(self, sim_paths: Iterable[str]) -> Dict[str, List[float]]: - """Collect values per scenario from the OutputManager.""" - - filtered_by_path: Dict[str, Dict[str, Any]] = { - path: self.om.filter_variables_pool({"filters": [path]}) for path in sim_paths - } - if not any(filtered_by_path.values()): - fallback_values = self._fallback_values_by_scenario(sim_paths) - return fallback_values + def _scenario_names(self) -> List[str]: + """Determine scenario names from the OutputManager variables pool. + Returns + ------- + list[str] + Scenario names found as top-level keys of the variables pool, or + ``["baseline"]`` when the pool is flat (plain variable payloads) + or empty. + """ scenario_names: List[str] = [] pool = getattr(self.om, "variables_pool", {}) if isinstance(pool, dict) and pool: @@ -253,7 +254,19 @@ def _fetch_values_by_scenario(self, sim_paths: Iterable[str]) -> Dict[str, List[ scenario_names = [name for name, data in pool.items() if isinstance(data, dict) and data] if not scenario_names: scenario_names = ["baseline"] + return scenario_names + + def _fetch_values_by_scenario(self, sim_paths: Iterable[str]) -> Dict[str, List[float]]: + """Collect values per scenario from the OutputManager.""" + + filtered_by_path: Dict[str, Dict[str, Any]] = { + path: self.om.filter_variables_pool({"filters": [path]}) for path in sim_paths + } + if not any(filtered_by_path.values()): + fallback_values = self._fallback_values_by_scenario(sim_paths) + return fallback_values + scenario_names = self._scenario_names() values_by_scenario: Dict[str, List[float]] = {scenario: [] for scenario in scenario_names} info_map = {"class": self.__class__.__name__, "function": self._fetch_values_by_scenario.__name__} @@ -866,59 +879,76 @@ def _extract_daily_seed_price(self, price_data: Any) -> list[float]: return daily_seed_price def _process_seed_costs_item(self) -> Dict[str, Any]: - """Build the full preprocessing result entry for the Seeds costs line item.""" + """Build the full preprocessing result entry for the Seeds costs line item. + + Scenario names are resolved from the OutputManager variables pool the + same way ``preprocess`` does for regular line items. Field schedules + and sizes come from the InputManager and do not vary by scenario, so + every scenario receives the same seed cost values. + """ info_map = {"class": self.__class__.__name__, "function": "_process_seed_costs_item"} - daily_area_by_seed = self._preprocess_seed_costs() + scenario_names = self._scenario_names() + - biophysical_values: dict[str, list[float]] = {} - bio_total: dict[str, float] = {} - price_data: dict[str, list[float]] = {} - price_values: dict[str, list[float]] = {} - price_aggregate: dict[str, float] = {} - total_seed_cost = 0.0 + biophysical_values_by_scenario: dict[str, dict[str, list[float]]] = {} + biophysical_aggregate_by_scenario: dict[str, dict[str, float]] = {} + line_item_values_by_scenario: dict[str, float] = {} - for seed_key, daily_area in daily_area_by_seed.items(): - raw_price = self._get_data_with_handling(seed_key, info_map) - if raw_price is None: + for scenario_name in scenario_names: + daily_area_by_seed = self._preprocess_seed_costs() + + biophysical_values: dict[str, list[float]] = {} + bio_total: dict[str, float] = {} + price_data: dict[str, list[float]] = {} + price_values: dict[str, list[float]] = {} + price_aggregate: dict[str, float] = {} + total_seed_cost = 0.0 + for seed_key, daily_area in daily_area_by_seed.items(): + raw_price = self._get_data_with_handling(seed_key, info_map) + if raw_price is None: + self.om.add_warning( + "MissingEconomicsFile", + f"Seed commodity pricing '{seed_key}' not found in InputManager", + info_map, + ) + continue + + extracted_prices = self._extract_daily_seed_price({seed_key: raw_price}) + if not extracted_prices: + continue + + daily_price_per_area = [ + seed_cost * area_m2 for seed_cost, area_m2 in zip(extracted_prices, daily_area) + ] + biophysical_values[seed_key] = daily_area_by_seed[seed_key] + price_values[seed_key] = extracted_prices + bio_total[seed_key] = sum(biophysical_values[seed_key]) + price_data[seed_key] = raw_price + price_aggregate[seed_key] = self._aggregate(extracted_prices, "average") + total_seed_cost += sum(daily_price_per_area) + + biophysical_values_by_scenario[scenario_name] = biophysical_values + biophysical_aggregate_by_scenario[scenario_name] = bio_total + line_item_values_by_scenario[scenario_name] = total_seed_cost + + if not daily_area_by_seed: self.om.add_warning( - "MissingEconomicsFile", - f"Seed commodity pricing '{seed_key}' not found in InputManager", + "MissingBiophysicalData", + "No field data found for seed cost preprocessing", info_map, ) - continue - - extracted_prices = self._extract_daily_seed_price({seed_key: raw_price}) - if not extracted_prices: - continue - - daily_price_per_area = [ - seed_cost * area_m2 for seed_cost, area_m2 in zip(extracted_prices, daily_area) - ] - biophysical_values[seed_key] = daily_area_by_seed[seed_key] - price_values[seed_key] = extracted_prices - bio_total[seed_key] = sum(biophysical_values[seed_key]) - price_data[seed_key] = raw_price - price_aggregate[seed_key] = self._aggregate(extracted_prices, "average") - total_seed_cost += sum(daily_price_per_area) - - if not daily_area_by_seed: - self.om.add_warning( - "MissingBiophysicalData", - "No field data found for seed cost preprocessing", - info_map, - ) return { "biophysical_values": biophysical_values, "biophysical_aggregate": bio_total, - "biophysical_values_by_scenario": {"baseline": biophysical_values}, - "biophysical_aggregate_by_scenario": {"baseline": bio_total}, + "biophysical_values_by_scenario": biophysical_values_by_scenario, + "biophysical_aggregate_by_scenario": biophysical_aggregate_by_scenario, "price_data": price_data, "price_values": price_values, "price_aggregate": price_aggregate, - "line_item_values_by_scenario": {"baseline": total_seed_cost}, + "line_item_values_by_scenario": line_item_values_by_scenario, "flow_type": "cost", } @@ -956,6 +986,8 @@ def preprocess(self) -> Dict[str, Dict[str, Dict[str, Dict[str, Any]]]]: if item.section == "Soil_and_crop" and item.name == "Seeds costs": category_data[item.name] = self._process_seed_costs_item() + #TODO: remove this + # json.dump(category_data[item.name], open(f"seed_cost_data.json", "w"), indent=4) continue values_by_scenario = self._fetch_values_by_scenario(item.biophysical_simulation) diff --git a/tests/test_EEE/test_economics_preprocessing.py b/tests/test_EEE/test_economics_preprocessing.py index f8810abd83..c3d741d1a6 100644 --- a/tests/test_EEE/test_economics_preprocessing.py +++ b/tests/test_EEE/test_economics_preprocessing.py @@ -730,3 +730,39 @@ def test_preprocess_seed_costs_clips_period_to_simulation_window(monkeypatch: py assert arr[299] == pytest.approx(expected_daily) assert arr[364] == pytest.approx(expected_daily) assert sum(arr) == pytest.approx(expected_sum) + + +def test_process_seed_costs_handles_multiple_scenarios(monkeypatch: pytest.MonkeyPatch) -> None: + """Seed cost entry carries one value per scenario found in the OM pool. + + Field schedules do not vary by scenario, so every scenario receives the + same seed cost total (1 ha corn, 100 days × 100 m²/day × $0.01/m² = $100). + """ + dummy_im = DummyInputManager( + data={ + "field_a": {"crop_specification": "RotA", "field_size": 1.0}, + "RotA.crop_schedules": [_corn_schedule(100, 200)], + "commodity_prices_corn_seed_dollar_per_square_meter": {"fips": [1001], "2020": [0.01]}, + }, + field_keys=["field_a"], + ) + dummy_om = DummyOutputManager({}) + dummy_om.variables_pool = { + "baseline": {"some.variable": {"values": [1.0]}}, + "scenario_1": {"some.variable": {"values": [2.0]}}, + } + monkeypatch.setattr(preprocessing, "InputManager", lambda: dummy_im) + monkeypatch.setattr(preprocessing, "OutputManager", lambda: dummy_om) + monkeypatch.setattr(preprocessing, "ECONOMIC_MAP", _seed_cost_map()) + + preprocessor = preprocessing.EconomicPreprocessor() + results = preprocessor.preprocess() + + item = results["Soil_and_crop"]["Costs"]["Seeds costs"] + seed_key = "commodity_prices_corn_seed_dollar_per_square_meter" + assert set(item["line_item_values_by_scenario"]) == {"baseline", "scenario_1"} + assert set(item["biophysical_values_by_scenario"]) == {"baseline", "scenario_1"} + assert set(item["biophysical_aggregate_by_scenario"]) == {"baseline", "scenario_1"} + for scenario in ("baseline", "scenario_1"): + assert item["line_item_values_by_scenario"][scenario] == pytest.approx(100.0) + assert item["biophysical_aggregate_by_scenario"][scenario][seed_key] == pytest.approx(10_000.0) From 7898cb23c8732199ac14020dab2536647be84650 Mon Sep 17 00:00:00 2001 From: Allister Liu Date: Mon, 13 Jul 2026 11:58:25 -0400 Subject: [PATCH 8/9] fallback --- RUFAS/EEE/EEE_manager.py | 1 + RUFAS/EEE/economics/fallback_values.py | 1 + RUFAS/EEE/economics/preprocessing.py | 17 ++++++++--------- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/RUFAS/EEE/EEE_manager.py b/RUFAS/EEE/EEE_manager.py index 016d15dfd2..b835482bed 100644 --- a/RUFAS/EEE/EEE_manager.py +++ b/RUFAS/EEE/EEE_manager.py @@ -31,6 +31,7 @@ def estimate_all() -> None: "Failed to load runtime metadata for 'EEE_econ'. Aborting emissions estimation.", info_map, ) + print("Failed to load runtime metadata for 'EEE_econ'. Aborting emissions estimation.") return om.add_log("Economics Processing", "Starting processing of economics.", info_map) EconomicFramework().run_economic_analysis() diff --git a/RUFAS/EEE/economics/fallback_values.py b/RUFAS/EEE/economics/fallback_values.py index 00e7fbe535..52fa246b4f 100644 --- a/RUFAS/EEE/economics/fallback_values.py +++ b/RUFAS/EEE/economics/fallback_values.py @@ -19,6 +19,7 @@ "field.crop_specification": [20.0], "field.field_size": [30.0], "Waiting on tractor_implement and other parts of EEE outputs": [6.0], + "seed_cost": [0.0], } # Default quantity used when no biophysical or input values are available. diff --git a/RUFAS/EEE/economics/preprocessing.py b/RUFAS/EEE/economics/preprocessing.py index 287ebf8fa7..55f6e9db67 100644 --- a/RUFAS/EEE/economics/preprocessing.py +++ b/RUFAS/EEE/economics/preprocessing.py @@ -235,7 +235,7 @@ def _fetch_values(self, sim_paths: Iterable[str]) -> List[float]: ) return values - def _scenario_names(self) -> List[str]: + def _scenario_names(self) -> list[str]: """Determine scenario names from the OutputManager variables pool. Returns @@ -245,7 +245,7 @@ def _scenario_names(self) -> List[str]: ``["baseline"]`` when the pool is flat (plain variable payloads) or empty. """ - scenario_names: List[str] = [] + scenario_names: list[str] = [] pool = getattr(self.om, "variables_pool", {}) if isinstance(pool, dict) and pool: if all(isinstance(value, dict) and "values" in value for value in pool.values()): @@ -793,9 +793,7 @@ def _preprocess_seed_costs(self) -> dict[str, List[float]]: crop_species = schedule.get("crop_species") if not isinstance(crop_species, str): continue - seed_key = self._CROP_TO_SEED_KEY.get(crop_species) - if seed_key is None: - continue + seed_key = self._CROP_TO_SEED_KEY.get(crop_species, f"fallback_{crop_species}") if seed_key not in daily_area_by_seed: daily_area_by_seed[seed_key] = [0.0] * total_sim_days @@ -850,6 +848,9 @@ def _extract_daily_seed_price(self, price_data: Any) -> list[float]: for key, value in price_data.items(): fallback_prices: list[float] | None = None price_by_year: dict[int, float] = {} + if "fallback" in key: + daily_seed_price = BIOPHYSICAL_FALLBACKS["seed_cost"] * days_count + break if not isinstance(value, dict) or "fips" not in value or not isinstance(value["fips"], list): self.om.add_warning( "MissingPriceData", @@ -891,7 +892,6 @@ def _process_seed_costs_item(self) -> Dict[str, Any]: scenario_names = self._scenario_names() - biophysical_values_by_scenario: dict[str, dict[str, list[float]]] = {} biophysical_aggregate_by_scenario: dict[str, dict[str, float]] = {} line_item_values_by_scenario: dict[str, float] = {} @@ -910,10 +910,9 @@ def _process_seed_costs_item(self) -> Dict[str, Any]: if raw_price is None: self.om.add_warning( "MissingEconomicsFile", - f"Seed commodity pricing '{seed_key}' not found in InputManager", + f"Seed commodity pricing '{seed_key}' not found in InputManager, using fallback price.", info_map, ) - continue extracted_prices = self._extract_daily_seed_price({seed_key: raw_price}) if not extracted_prices: @@ -987,7 +986,7 @@ def preprocess(self) -> Dict[str, Dict[str, Dict[str, Dict[str, Any]]]]: if item.section == "Soil_and_crop" and item.name == "Seeds costs": category_data[item.name] = self._process_seed_costs_item() #TODO: remove this - # json.dump(category_data[item.name], open(f"seed_cost_data.json", "w"), indent=4) + json.dump(category_data[item.name], open(f"seed_cost_data.json", "w"), indent=4) continue values_by_scenario = self._fetch_values_by_scenario(item.biophysical_simulation) From 1f3ba02367f99413004a14d2c0254be4027d5b3d Mon Sep 17 00:00:00 2001 From: Allister Liu Date: Tue, 21 Jul 2026 15:48:28 +0800 Subject: [PATCH 9/9] fix daily area when clipped --- RUFAS/EEE/economics/preprocessing.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/RUFAS/EEE/economics/preprocessing.py b/RUFAS/EEE/economics/preprocessing.py index 8f297c93f2..ced98b88e2 100644 --- a/RUFAS/EEE/economics/preprocessing.py +++ b/RUFAS/EEE/economics/preprocessing.py @@ -723,7 +723,7 @@ def _growing_periods( plant_date = None return periods - def _preprocess_seed_costs(self) -> dict[str, List[float]]: + def _preprocess_seed_costs(self) -> dict[str, list[float]]: """Build a daily time-series of responsible field area (m²) per seed key. For each field, each crop's planting-to-kill periods are located within @@ -767,7 +767,7 @@ def _preprocess_seed_costs(self) -> dict[str, List[float]]: ) return {} - daily_area_by_seed: dict[str, List[float]] = {} + daily_area_by_seed: dict[str, list[float]] = {} for field_key in field_keys: field_data = self.im.get_data(field_key) @@ -799,10 +799,9 @@ def _preprocess_seed_costs(self) -> dict[str, List[float]]: continue seed_key = self._CROP_TO_SEED_KEY.get(crop_species, f"fallback_{crop_species}") - if seed_key not in daily_area_by_seed: + if seed_key not in daily_area_by_seed.keys(): daily_area_by_seed[seed_key] = [0.0] * total_sim_days - arr = daily_area_by_seed[seed_key] growing_periods = self._growing_periods(schedule) for plant_date, kill_date in growing_periods: plant_idx = (plant_date - start_date).days @@ -814,10 +813,10 @@ def _preprocess_seed_costs(self) -> dict[str, List[float]]: if clipped_start >= clipped_end: continue - duration = kill_idx - plant_idx + duration = clipped_end - clipped_start daily_value = field_size_m2 / duration for i in range(clipped_start, clipped_end): - arr[i] += daily_value + daily_area_by_seed[seed_key][i] += daily_value return daily_area_by_seed