diff --git a/docs/release-notes.md b/docs/release-notes.md index 1f1f48a0..27b0f935 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -8,7 +8,7 @@ SPDX-License-Identifier: MIT ## Upcoming Version -* +* Add the [JRC Open Power Plants Database](https://doi.org/10.5281/zenodo.3574566) (`pm.data.JRC_PPDB_OPEN()`). Its coordinates fill the ENTSOE data via EIC codes, which helps to match ENTSOE with other datasets. In the matched output, other sources' coordinates take priority over these. ([PR #306](https://github.com/PyPSA/powerplantmatching/pull/306)) ## [v0.9.0](https://github.com/PyPSA/powerplantmatching/releases/tag/v0.9.0) (3rd October 2026) diff --git a/powerplantmatching/data.py b/powerplantmatching/data.py index 219f0a2a..205ef99c 100644 --- a/powerplantmatching/data.py +++ b/powerplantmatching/data.py @@ -484,6 +484,60 @@ def set_large_spanish_stores_to_reservoirs(df): return df +def JRC_PPDB_OPEN(raw=False, update=False, config=None): + """ + Importer for the JRC Open Power Plants Database (JRC-PPDB-OPEN, + Kanellopoulos et al., 2019, doi:10.5281/zenodo.3574566). + + The database extends the ENTSO-E production units with coordinates, + which are used to fill the coordinates of the ENTSOE data. Generation + units are aggregated to production units, which have the same EIC codes + as the ENTSOE data. + + Parameters + ---------- + raw : bool, default False + Whether to return the original dataset + update : bool, default False + Whether to update the data from the URL + config : dict, default None + Custom configuration + """ + config = get_config() if config is None else config + + fn = get_raw_file("JRC_PPDB_OPEN", update, config) + + with ZipFile(fn) as zf, zf.open("JRC_OPEN_UNITS.csv") as f: + df = pd.read_csv(f) + + if raw: + return df + + status_list = config["JRC_PPDB_OPEN"].get("status", ["COMMISSIONED"]) # noqa: F841 + + return ( + df.assign(status_g=df.status_g.str.upper()) + .query("status_g in @status_list") + .groupby("eic_p") + .agg( + Name=("name_p", "first"), + Fueltype=("type_g", "first"), + Country=("country", "first"), + Capacity=("capacity_g", "sum"), + DateIn=("year_commissioned", "min"), + lat=("lat", "mean"), + lon=("lon", "mean"), + ) + .rename_axis("projectID") + .reset_index() + .assign(EIC=lambda df: df.projectID, Technology=np.nan, Set=np.nan) + .pipe(gather_specifications, config=config) + .pipe(clean_name) + .pipe(set_column_name, "JRC_PPDB_OPEN") + .pipe(config_filter, config) + ) + + @deprecated( deprecated_in="0.5.0", details="Use the JRC data instead", @@ -688,7 +742,7 @@ def ENTSOE( ): """ Importer for the list of installed generators provided by the ENTSO-E - Transparency Project. Geographical information is not given. + Transparency Project. Coordinates are taken from JRC-PPDB-OPEN via EIC codes. If update=True, the dataset is parsed through a request to 'https://transparency.entsoe.eu/generation/r2/\ installedCapacityPerProductionUnit/show', @@ -778,6 +832,7 @@ def retrieve_data(token): fn = _package_data("entsoe_country_codes.csv") COUNTRY_MAP = pd.read_csv(fn, index_col=0).rename(index=str).Country + coords = JRC_PPDB_OPEN(config=config).set_index("projectID") return ( df.rename_axis(index="projectID") @@ -791,8 +846,8 @@ def retrieve_data(token): Capacity=lambda df: pd.to_numeric(df.Capacity), Technology=np.nan, Set=np.nan, - lat=np.nan, - lon=np.nan, + lat=lambda df: df.projectID.map(coords.lat), + lon=lambda df: df.projectID.map(coords.lon), ) .powerplant.convert_alpha2_to_country() # .pipe(fill_geoposition, **fill_geoposition_kwargs) diff --git a/powerplantmatching/matching.py b/powerplantmatching/matching.py index e1933d9f..7855e884 100644 --- a/powerplantmatching/matching.py +++ b/powerplantmatching/matching.py @@ -297,6 +297,12 @@ def reduce_matched_dataframe(df, show_orig_names=False, config=None): ) props_for_groups = pd.Series(props_for_groups)[cols].to_dict() + # ENTSOE coordinates are filled from JRC-PPDB-OPEN, use them only as fallback + if "ENTSOE" in sources: + df = df.copy() + others = df.lat.drop(columns="ENTSOE").notna().any(axis=1) + df.loc[others, [("lat", "ENTSOE"), ("lon", "ENTSOE")]] = np.nan + # set low priority on Fueltype 'Other' and Set 'PP' # turn it since aggregating only possible for axis=0 sdf = ( diff --git a/powerplantmatching/package_data/config.yaml b/powerplantmatching/package_data/config.yaml index 290fc401..9a5ba94e 100644 --- a/powerplantmatching/package_data/config.yaml +++ b/powerplantmatching/package_data/config.yaml @@ -97,6 +97,10 @@ JRC: reliability_score: 5 fn: jrc-hydro-power-plant-database.csv url: https://raw.githubusercontent.com/energy-modelling-toolkit/hydro-power-database/3e8a837/data/jrc-hydro-power-plant-database.csv +JRC_PPDB_OPEN: + status: ["COMMISSIONED", "RESERVE", "MOTHBALLED", "CONSTRUCTION"] + fn: JRC-PPDB-OPEN.ver1.0.zip + url: https://zenodo.org/records/3574566/files/JRC-PPDB-OPEN.ver1.0.zip GEO: net_capacity: false reliability_score: 2 diff --git a/test/test_data.py b/test/test_data.py index d3614371..669e6024 100755 --- a/test/test_data.py +++ b/test/test_data.py @@ -36,6 +36,10 @@ def test_data_request_processed(source): assert df.columns.to_list() == config["target_columns"] +def test_ENTSOE_coordinates(): + assert data.ENTSOE().lat.notna().mean() > 0.5 + + def test_OPSD_VRE(): df = pm.data.OPSD_VRE() assert not df.empty diff --git a/test/test_matching.py b/test/test_matching.py new file mode 100644 index 00000000..8669b895 --- /dev/null +++ b/test/test_matching.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Contributors to powerplantmatching +# +# SPDX-License-Identifier: MIT + +import numpy as np +import pandas as pd +import pytest + +import powerplantmatching as pm +from powerplantmatching.matching import reduce_matched_dataframe + + +@pytest.mark.parametrize("gpd_lat, expected", [(50.0, 50.0), (np.nan, 49.0)]) +def test_entsoe_coordinates_are_fallback(gpd_lat, expected): + plant = {"Name": "Plant", "Fueltype": "Hydro", "Country": "Austria", "lon": 10.0} + df = pd.concat( + { + "ENTSOE": pd.DataFrame([{**plant, "Capacity": 100.0, "lat": 49.0}]), + "GPD": pd.DataFrame([{**plant, "Capacity": 90.0, "lat": gpd_lat}]), + }, + axis=1, + ).swaplevel(axis=1) + cols = pd.MultiIndex.from_product( + [pm.get_config()["target_columns"], ["ENTSOE", "GPD"]] + ) + df = df.reindex(columns=cols) + reduced = reduce_matched_dataframe(df) + assert reduced.loc[0, "lat"] == expected + assert reduced.loc[0, "Capacity"] == 100.0