From 7191063b03677f37f37db5f81d86851b68492ee8 Mon Sep 17 00:00:00 2001 From: Wei Ai Date: Wed, 28 Oct 2026 01:25:42 -0400 Subject: [PATCH] Switch the fuel costs of the cost data from AEO2023 to AEO2026 - build_cost_data read the electric sector fuel costs of AEO2023 from PUDL, in 2022 dollars it did not convert. It now reads the AEO2026 Table 3 electric power prices of natural gas, steam coal and distillate fuel oil from the EIA API (eia.ProjectedFuelCost), and converts them from 2025$/MMBtu to 2023$/MWh_th. PUDL's AEO2026 table holds only nominal prices. - costs: aeo: scenario takes the AEO2026 cases. AEO2026 has no Reference case, so reference is its Counterfactual Baseline, and existing configs keep validating. Co-Authored-By: Claude Opus 5.5 --- docs/source/configtables/costs.csv | 4 +- workflow/repo_data/config/config.default.yaml | 4 +- workflow/rules/build_electricity.smk | 1 + workflow/schemas/config.schema.yaml | 3 +- workflow/scripts/build_cost_data.py | 54 ++++++------ workflow/scripts/eia.py | 85 ++++++++++++++++++ workflow/scripts/test/test_aeo_fuel_costs.py | 87 +++++++++++++++++++ 7 files changed, 205 insertions(+), 33 deletions(-) create mode 100644 workflow/scripts/test/test_aeo_fuel_costs.py diff --git a/docs/source/configtables/costs.csv b/docs/source/configtables/costs.csv index 18c67fdd..9da30372 100644 --- a/docs/source/configtables/costs.csv +++ b/docs/source/configtables/costs.csv @@ -3,8 +3,8 @@ atb,,,"NREL ATB scenario selection. ATB carries every (scenario, model_case) for -- scenario,--,``Moderate`` \| ``Advanced`` \| ``Conservative``,"Global default ATB cost trajectory. ``Advanced`` = optimistic / fastest cost decline; ``Conservative`` = pessimistic. See :ref:`data-costs`." -- model_case,--,``R&D + TC`` \| ``R&D`` \| ``Exp + TC`` \| ``Exp``,"Global default ATB 2025 case. ``R&D`` = research-and-development cost drivers, ``Exp`` = expanded cost drivers (published for utility PV, land-based wind and utility-scale batteries only; other carriers fall back to the matching ``R&D`` case). ``+ TC`` adds the tax credits enacted as of July 2025 to the financing assumptions." -- overrides,,Mapping,"Optional per-carrier overrides. Keys are pypsa-names (``solar``, ``onwind``, ``nuclear``, ``CCGT``, ``4hr_battery_storage``, …); each value may set ``scenario`` and/or ``model_case``. Anything unset falls back to the global default." -aeo,,,"EIA AEO fuel-cost scenario." --- scenario,--,``reference`` \| ``high`` \| ``low``,"AEO case used to fetch projected fuel costs for the requested year." +aeo,,,"EIA Annual Energy Outlook 2026 case of the fuel costs of gas, coal and oil plants." +-- scenario,--,``reference`` \| ``high_oil_and_gas_supply`` \| ``low_oil_and_gas_supply`` \| ``high_economic_growth`` \| ``low_economic_growth`` \| ``high_zero_carbon_technology_cost`` \| ``low_zero_carbon_technology_cost`` \| ``high_electricity_demand`` \| ``alternative_electricity`` \| ``alternative_transportation`` \| ``alternative_electricity_and_transportation``,"AEO2026 case of the electric power prices of natural gas, steam coal and distillate fuel oil (Table 3) for the requested year. AEO2026 has no Reference case; ``reference`` is its Counterfactual Baseline." social_discount_rate,--,float; e.g. ``0.02``,"Discount rate used for weighting multi-investment objective function values." ng_fuel_year,--,int; one of 2019-2023,"Vintage year of the CAISO natural-gas hub prices used for wholesale gas costs." emission_prices,,,"Specify exogenous prices for emission types listed in ``network.carriers`` to marginal costs." diff --git a/workflow/repo_data/config/config.default.yaml b/workflow/repo_data/config/config.default.yaml index a6b3eeba..e236a420 100644 --- a/workflow/repo_data/config/config.default.yaml +++ b/workflow/repo_data/config/config.default.yaml @@ -346,9 +346,9 @@ costs: # scenario: "Conservative" # model_case: "R&D" - # EIA Annual Energy Outlook — used for fuel-price escalation + # EIA Annual Energy Outlook 2026 — electric power fuel prices of gas, coal and oil plants aeo: - scenario: "reference" # reference | high | low + scenario: "reference" # AEO2026 case; reference (= Counterfactual Baseline) | high_oil_and_gas_supply | low_oil_and_gas_supply | ... social_discount_rate: 0.02 # fraction; applied to annualization of capex ng_fuel_year: 2019 # CAISO NG price vintage year; 2019..2023 diff --git a/workflow/rules/build_electricity.smk b/workflow/rules/build_electricity.smk index 305974a1..9c3b7aa8 100644 --- a/workflow/rules/build_electricity.smk +++ b/workflow/rules/build_electricity.smk @@ -131,6 +131,7 @@ rule build_bus_regions: rule build_cost_data: params: aeo=config_provider("costs", "aeo"), + eia_api=config_provider("api", "eia"), pudl_path=config_provider("pudl_path"), input: efs_tech_costs="repo_data/costs/EFS_Technology_Data.xlsx", diff --git a/workflow/schemas/config.schema.yaml b/workflow/schemas/config.schema.yaml index 0c02fb18..21adfda8 100644 --- a/workflow/schemas/config.schema.yaml +++ b/workflow/schemas/config.schema.yaml @@ -395,7 +395,8 @@ properties: type: object additionalProperties: false properties: - scenario: {enum: [reference, high, low]} + scenario: + enum: [reference, high_oil_and_gas_supply, low_oil_and_gas_supply, high_economic_growth, low_economic_growth, high_zero_carbon_technology_cost, low_zero_carbon_technology_cost, high_electricity_demand, alternative_electricity, alternative_transportation, alternative_electricity_and_transportation] social_discount_rate: {type: number} ng_fuel_year: {type: integer} emission_prices: diff --git a/workflow/scripts/build_cost_data.py b/workflow/scripts/build_cost_data.py index 04f9fb72..bfad5427 100644 --- a/workflow/scripts/build_cost_data.py +++ b/workflow/scripts/build_cost_data.py @@ -11,6 +11,7 @@ EfsTechnologyData, EiaBuildingData, ) +from eia import ProjectedFuelCost logger = logging.getLogger(__name__) @@ -110,16 +111,6 @@ def load_pudl_atb_data(parquet_path: str): return duckdb.query(query).to_df() -def load_pudl_aeo_data(parquet_path: str): - """Loads AEO data directly from parquet files.""" - query = f""" - SELECT * - FROM read_parquet('{parquet_path}/core_eiaaeo__yearly_projected_fuel_cost_in_electric_sector_by_type.parquet') aeo - WHERE aeo.report_year = 2023 - """ - return duckdb.query(query).to_df() - - def match_technology(row, tech_dict): for key, value in tech_dict.items(): # Match technology and techdetail @@ -308,23 +299,30 @@ def calculate_capex(df: pd.DataFrame, discount_rate: float) -> pd.DataFrame: ] -def build_aeo_fuel_costs(parquet_path: str, tech_year: int, aeo_scenario: str) -> pd.DataFrame: - """Loads AEO fuel cost data, applies tech mappings, returns long-format frame.""" - aeo = load_pudl_aeo_data(parquet_path) - aeo = aeo[aeo.projection_year == tech_year] - aeo = aeo[aeo.model_case_eiaaeo == aeo_scenario] - cols = ["fuel_type_eiaaeo", "fuel_cost_real_per_mmbtu_eiaaeo"] - aeo = aeo[cols] - aeo = aeo.groupby("fuel_type_eiaaeo").mean() - aeo["fuel_cost_real_per_mwhth"] = aeo["fuel_cost_real_per_mmbtu_eiaaeo"] * 3.412 - aeo = pd.melt( - aeo.reset_index(), - id_vars="fuel_type_eiaaeo", - value_vars=["fuel_cost_real_per_mwhth"], - var_name="parameter", - value_name="value", - ) - aeo = aeo.rename(columns={"fuel_type_eiaaeo": "pypsa-name"}) +def build_aeo_fuel_costs(api: str, tech_year: int, aeo_scenario: str) -> pd.DataFrame: + """ + Loads AEO fuel cost data, applies tech mappings, returns long-format frame. + + The fuel costs are the AEO2026 electric power prices of the tech year (see + ``eia.ProjectedFuelCost``), converted from 2025$/MMBtu to 2023$/MWh_th. + Years outside the AEO years take the price of the nearest year. + """ + rows = [] + for name, fuel in {"natural_gas": "gas", "coal": "coal", "distillate_fuel_oil": "oil"}.items(): + df = ProjectedFuelCost(fuel, "power", api, aeo_scenario).get_data() + if not (df.units == "2025 $/MMBtu").all(): + raise ValueError(f"Expected AEO {fuel} prices in 2025 $/MMBtu, got {df.units.unique().tolist()}") + year = min(df.index, key=lambda x: abs(x - tech_year)) + if year != tech_year: + logger.warning(f"No AEO2026 {fuel} price for {tech_year}; using {year}") + rows.append( + { + "pypsa-name": name, + "parameter": "fuel_cost_real_per_mwhth", + "value": df.value.at[year] * const.USD_2025_2_USD_2023 * 3.412, + }, + ) + aeo = pd.DataFrame(rows) addnl_fuels = pd.DataFrame( [ @@ -567,7 +565,7 @@ def process_atb_scenario(pudl_atb_sub: pd.DataFrame, aeo: pd.DataFrame) -> pd.Da # Build AEO fuel cost data once — independent of ATB scenario / case aeo = build_aeo_fuel_costs( - parquet_path, + snakemake.params.eia_api, tech_year, aeo_params.get("scenario", "reference"), ) diff --git a/workflow/scripts/eia.py b/workflow/scripts/eia.py index ae04eabb..5d391750 100644 --- a/workflow/scripts/eia.py +++ b/workflow/scripts/eia.py @@ -372,6 +372,28 @@ def data_creator(self) -> pd.DataFrame: ) +# concrete creator +class ProjectedFuelCost(EiaData): + """Fuel price projections by sector.""" + + def __init__(self, fuel: str, sector: str, api: str, scenario: str = "reference") -> None: + self.fuel = fuel # (gas|coal|oil) + self.sector = sector # (power) + self.api = api + self.scenario = scenario # AEO case (see _PowerFuelCost.scenario_codes) + + def data_creator(self) -> pd.DataFrame: + """Initializes data extractor.""" + if self.sector == "power": + return _PowerFuelCost(self.fuel, self.scenario, self.api) + else: + raise InputPropertyError( + propery="Projected Fuel Cost", + valid_options=["power"], + recived_option=self.sector, + ) + + # concrete creator class EnergyDemand(EiaData): """ @@ -1539,6 +1561,69 @@ def format_data(self, df: pd.DataFrame) -> pd.DataFrame: return self._assign_dtypes(df) +class _PowerFuelCost(DataExtractor): + """ + Annual fuel price projections of the electric power sector. + + Electric power prices of AEO2026 Table 3 (Energy Prices by Sector and + Source), United States, in 2025$/MMBtu: natural gas, steam coal and + distillate fuel oil. AEO2026 has no Reference case, so ``reference`` is + its Counterfactual Baseline case. + """ + + scenario_codes: ClassVar[dict[str, str]] = { + "reference": "cb2026", # Counterfactual Baseline + "high_oil_and_gas_supply": "highogs", + "low_oil_and_gas_supply": "lowogs", + "high_economic_growth": "hm2026", + "low_economic_growth": "lm2026", + "high_zero_carbon_technology_cost": "highZTC", + "low_zero_carbon_technology_cost": "lowZTC", + "high_electricity_demand": "higheldmd", + "alternative_electricity": "altelec", + "alternative_transportation": "alttrnp", + "alternative_electricity_and_transportation": "electrnp", + } + + fuel_codes: ClassVar[dict[str, str]] = { + "gas": "ng", # natural gas + "coal": "stc", # steam coal + "oil": "dfo", # distillate fuel oil + } + + def __init__(self, fuel: str, scenario: str, api_key: str) -> None: + if fuel not in self.fuel_codes: + raise InputPropertyError( + propery="Power Fuel Cost", + valid_options=list(self.fuel_codes), + recived_option=fuel, + ) + if scenario not in self.scenario_codes: + raise InputPropertyError( + propery="AEO Scenario", + valid_options=list(self.scenario_codes), + recived_option=scenario, + ) + self.fuel = fuel + self.scenario = scenario + super().__init__(year=None, api_key=api_key) # every AEO year is returned + + def build_url(self) -> str: + series_id = f"prce_real_elep_NA_{self.fuel_codes[self.fuel]}_NA_NA_y13dlrpmmbtu" + base_url = "aeo/2026/data/" + facets = f"frequency=annual&data[0]=value&facets[scenario][]={self.scenario_codes[self.scenario]}&facets[seriesId][]={series_id}&sort[0][column]=period&sort[0][direction]=asc&offset=0&length=5000" + return f"{API_BASE}{base_url}?api_key={self.api_key}&{facets}" + + def format_data(self, df: pd.DataFrame) -> pd.DataFrame: + df.index = df.period.astype(int) + df = df.rename( + columns={"seriesName": "series-description", "unit": "units"}, + ) + df["state"] = "USA" + df = df[["series-description", "value", "units", "state"]].sort_index() + return self._assign_dtypes(df) + + class _GasBalance(DataExtractor): """ Annual U.S. total of one item of the natural gas supply and disposition balance. diff --git a/workflow/scripts/test/test_aeo_fuel_costs.py b/workflow/scripts/test/test_aeo_fuel_costs.py new file mode 100644 index 00000000..998c0fc4 --- /dev/null +++ b/workflow/scripts/test/test_aeo_fuel_costs.py @@ -0,0 +1,87 @@ +""" +Test the AEO fuel costs of the cost data. + +``build_cost_data`` reads the AEO2026 electric power prices of natural gas, +steam coal and distillate fuel oil (Table 3) for the case of ``costs: aeo: +scenario`` from the EIA API, in 2025$/MMBtu, and writes them in 2023$/MWh_th. +The EIA layer is patched throughout. +""" + +import os +import sys + +import pandas as pd +import pytest + +sys.path.append(os.path.join(os.path.dirname(__file__), "..")) + +import eia +from build_cost_data import build_aeo_fuel_costs + +# 2025$/MMBtu of each fuel in 2025, 2030 and 2050 +AEO_PRICES = {"gas": [4.0, 5.0, 6.0], "coal": [2.0, 2.5, 3.0], "oil": [20.0, 21.0, 22.0]} +TO_MWH = 3.412 / 1.06 # 2025$/MMBtu -> 2023$/MWh_th + + +@pytest.fixture +def patch_prices(monkeypatch): + def _patch(units: str = "2025 $/MMBtu") -> None: + def get_data(self, pivot=False): + return pd.DataFrame( + {"series-description": self.fuel, "value": AEO_PRICES[self.fuel], "units": units, "state": "USA"}, + index=pd.Index([2025, 2030, 2050], name="period"), + ) + + monkeypatch.setattr(eia.ProjectedFuelCost, "get_data", get_data) + + return _patch + + +def fuel_costs(aeo: pd.DataFrame) -> pd.Series: + return aeo[aeo.parameter == "fuel_cost_real_per_mwhth"].set_index("pypsa-name").value + + +def test_url_targets_aeo2026_electric_power_prices(): + url = eia.ProjectedFuelCost("coal", "power", "KEY").get_api_call() + + assert url.startswith(f"{eia.API_BASE}aeo/2026/data/?api_key=KEY&") + # AEO2026 has no Reference case: reference is the Counterfactual Baseline + assert "facets[scenario][]=cb2026" in url + assert "facets[seriesId][]=prce_real_elep_NA_stc_NA_NA_y13dlrpmmbtu" in url + + +def test_other_aeo2026_cases(): + url = eia.ProjectedFuelCost("gas", "power", "KEY", "high_oil_and_gas_supply").get_api_call() + + assert "facets[scenario][]=highogs" in url + assert "facets[seriesId][]=prce_real_elep_NA_ng_NA_NA_y13dlrpmmbtu" in url + + +def test_unknown_case_is_rejected(): + with pytest.raises(eia.InputPropertyError): + eia.ProjectedFuelCost("gas", "power", "KEY", "high").get_api_call() + + +def test_fuel_costs_are_2023_usd_per_mwh_th(patch_prices): + patch_prices() + + costs = fuel_costs(build_aeo_fuel_costs("KEY", 2030, "reference")) + + assert costs["natural_gas"] == pytest.approx(5.0 * TO_MWH) + assert costs["coal"] == pytest.approx(2.5 * TO_MWH) + assert costs["distillate_fuel_oil"] == pytest.approx(21.0 * TO_MWH) + assert costs[["CCGT", "OCGT", "CCGT-95CCS", "CCGT-97CCS"]].tolist() == pytest.approx([5.0 * TO_MWH] * 4) + assert costs[["coal-95CCS", "coal-99CCS"]].tolist() == pytest.approx([2.5 * TO_MWH] * 2) + + +def test_years_outside_the_aeo_take_the_nearest_year(patch_prices): + patch_prices() + + assert fuel_costs(build_aeo_fuel_costs("KEY", 2024, "reference"))["natural_gas"] == pytest.approx(4.0 * TO_MWH) + + +def test_other_dollar_year_is_rejected(patch_prices): + patch_prices(units="2024 $/MMBtu") + + with pytest.raises(ValueError, match="2025 \\$/MMBtu"): + build_aeo_fuel_costs("KEY", 2030, "reference")