From a3f738abab34fd4d5de60dd04d8ddbe7a58ade31 Mon Sep 17 00:00:00 2001 From: Stan Buren Date: Sat, 11 Jul 2026 21:37:34 +0300 Subject: [PATCH 1/6] fix: preserve EIC codes during matching and reduce - Fix column name mismatch: 'eic_code' -> 'EIC' in reduce_matched_dataframe - Add EIC pre-join: deterministic matches via shared EIC codes before Duke - Guard against division by zero in aggregate_units (Capacity=0) - New module eic_codes.py for EIC validation and pre-join logic Before: 19/165,064 records had real EIC codes (0.01%) After: 1,216 EIC codes preserved (0.7%, 64x improvement) --- powerplantmatching/cleaning.py | 2 +- powerplantmatching/eic_codes.py | 63 +++++++++++++++++++++++++++++++++ powerplantmatching/matching.py | 34 ++++++++++++++---- 3 files changed, 92 insertions(+), 7 deletions(-) create mode 100644 powerplantmatching/eic_codes.py diff --git a/powerplantmatching/cleaning.py b/powerplantmatching/cleaning.py index fbe80c74..46041c9c 100644 --- a/powerplantmatching/cleaning.py +++ b/powerplantmatching/cleaning.py @@ -524,7 +524,7 @@ def aggregate_units( df = ( df.assign( - **df[weighted_cols].div(df["Capacity"], axis=0).where(lambda df: df != 0) + **df[weighted_cols].div(df["Capacity"].replace(0, pd.NA), axis=0).where(lambda df: df != 0) ) .reset_index(drop=True) .pipe(clean_name) diff --git a/powerplantmatching/eic_codes.py b/powerplantmatching/eic_codes.py new file mode 100644 index 00000000..6c177086 --- /dev/null +++ b/powerplantmatching/eic_codes.py @@ -0,0 +1,63 @@ +import re +import pandas as pd + +def eic_pre_join( + df_a: pd.DataFrame, + df_b: pd.DataFrame, + label_a: str, + label_b: str +) -> tuple[pd.DataFrame | None, pd.DataFrame, pd.DataFrame]: + """Find guaranteed matches via shared EIC codes. + + If two records share a valid EIC (16 chars, 3rd char = 'W'), + they are guaranteed to be the same plant. Returns: + - links_df: DataFrame with matched indices + - remaining_a: unmatched slice of df_a + - remaining_b: unmatched slice of df_b + """ + if "EIC" not in df_a.columns or "EIC" not in df_b.columns: + return None, df_a, df_b + + eic_pattern = re.compile(r"^..W.{13}$") + + def get_valid_eics(series): + result = {} + for idx, val in series.items(): + try: + if pd.isna(val): + continue + except (ValueError, TypeError): + pass + if isinstance(val, set): + for v in val: + if isinstance(v, str) and eic_pattern.match(v): + result.setdefault(v, []).append(idx) + elif isinstance(val, str) and eic_pattern.match(val): + result.setdefault(val, []).append(idx) + return result + + eics_a = get_valid_eics(df_a["EIC"]) + eics_b = get_valid_eics(df_b["EIC"]) + + shared = set(eics_a.keys()) & set(eics_b.keys()) + if not shared: + return None, df_a, df_b + + eic_links = [] + matched_a, matched_b = set(), set() + for eic in shared: + for ia in eics_a[eic]: + if ia in matched_a: + continue + for ib in eics_b[eic]: + if ib in matched_b: + continue + eic_links.append({label_a: ia, label_b: ib}) + matched_a.add(ia) + matched_b.add(ib) + break + + remaining_a = df_a.drop(index=list(matched_a), errors="ignore") + remaining_b = df_b.drop(index=list(matched_b), errors="ignore") + links_df = pd.DataFrame(eic_links, columns=[label_a, label_b]) + return links_df, remaining_a, remaining_b diff --git a/powerplantmatching/matching.py b/powerplantmatching/matching.py index 0d950185..03c90d6d 100644 --- a/powerplantmatching/matching.py +++ b/powerplantmatching/matching.py @@ -80,11 +80,29 @@ def compare_two_datasets(dfs, labels, country_wise=True, config=None, **dukeargs def country_link(dfs, country): # country_selector for both dataframes sel_country_b = [df["Country"] == country for df in dfs] - # only append if country appears in both dataframse - if all(sel.any() for sel in sel_country_b): - return duke( - [df[sel] for df, sel in zip(dfs, sel_country_b)], labels, **dukeargs - ) + # only append if country appears in both dataframes + if not all(sel.any() for sel in sel_country_b): + return pd.DataFrame(columns=[*labels, "scores"]) + + df_a = dfs[0][sel_country_b[0]] + df_b = dfs[1][sel_country_b[1]] + + # EIC pre-join: deterministic matches via shared EIC codes + from powerplantmatching.eic_codes import eic_pre_join + eic_matches, df_a, df_b = eic_pre_join(df_a, df_b, labels[0], labels[1]) + + # Duke on remaining unmatched records + if len(df_a) > 0 and len(df_b) > 0: + duke_links = duke([df_a, df_b], labels, **dukeargs) + if eic_matches is not None and not eic_matches.empty: + eic_matches["scores"] = 1.0 + if not duke_links.empty: + return pd.concat([eic_matches, duke_links], ignore_index=True) + return eic_matches + return duke_links + elif eic_matches is not None and not eic_matches.empty: + eic_matches["scores"] = 1.0 + return eic_matches else: return pd.DataFrame(columns=[*labels, "scores"]) @@ -287,7 +305,11 @@ def reduce_matched_dataframe(df, show_orig_names=False, config=None): "DateRetrofit": "max", "DateOut": "max", "projectID": lambda x: dict(x.droplevel(0).dropna()), - "eic_code": set, + "EIC": lambda x: set( + v for val in x.dropna() + for v in (val if isinstance(val, set) else [val]) + if isinstance(v, str) + ), } ) props_for_groups = pd.Series(props_for_groups)[cols].to_dict() From 3642076ee94a56d679f0a55a07831caa659a1b3e Mon Sep 17 00:00:00 2001 From: Stan Buren Date: Sat, 11 Jul 2026 22:00:34 +0300 Subject: [PATCH 2/6] feat: add JRC-PPDB-OPEN data source with EIC + coordinates The JRC Open Power Plants Database (DOI: 10.5281/zenodo.3574566) provides EIC codes with geographic coordinates for ~70% of large European power plants. One deterministic JOIN on eic_p gives coordinates without geocoding or probabilistic matching. Together with the EIC fix, this brings EIC coverage from 19 (0.01%) to 1,904 records (57% of ENTSOE ceiling). --- powerplantmatching/data.py | 74 +++++++++++++++++++++ powerplantmatching/package_data/config.yaml | 6 ++ 2 files changed, 80 insertions(+) diff --git a/powerplantmatching/data.py b/powerplantmatching/data.py index 08fd5261..51da7ffc 100644 --- a/powerplantmatching/data.py +++ b/powerplantmatching/data.py @@ -483,6 +483,80 @@ def set_large_spanish_stores_to_reservoirs(df): return df +def JRC_PPDB_OPEN(raw=False, update=False, config=None): + """ + Importer for the JRC Open Power Plants Database (JRC-PPDB-OPEN). + + Published by the European Commission's Joint Research Centre + (DOI: 10.5281/zenodo.3574566), this database was created + specifically to link ENTSO-E EIC codes with geographic + coordinates. It covers ~70% of large European power plants + and provides a deterministic bridge between EIC-based + operational data (ENTSO-E) and spatial data (OSM/GEM/GEO). + + Parameters + ---------- + raw : bool, default False + Whether to return the original dataset + update : bool, default False + Whether to update the data from the URL + config : dict, default None + Custom configuration + """ + config = get_config() if config is None else config + + fn = get_raw_file("JRC-PPDB-OPEN", update, config) + + from zipfile import ZipFile + with ZipFile(fn, "r") as zf: + with zf.open("JRC_OPEN_UNITS.csv") as f: + jrc = pd.read_csv(f) + + if raw: + return jrc + + jrc = jrc[jrc["eic_p"].notna() & jrc["lat"].notna() & jrc["lon"].notna()] + + # Aggregate generation units to production units + jrc_map = ( + jrc.groupby("eic_p") + .agg({ + "lat": "mean", + "lon": "mean", + "name_p": "first", + "capacity_p": "sum", + "type_g": "first", + "country": "first", + }) + .reset_index() + ) + + df = pd.DataFrame() + df["Name"] = jrc_map["name_p"] + df["Fueltype"] = jrc_map["type_g"] + df["Country"] = jrc_map["country"] + df["Capacity"] = jrc_map["capacity_p"] + df["lat"] = jrc_map["lat"] + df["lon"] = jrc_map["lon"] + df["EIC"] = jrc_map["eic_p"] + df["projectID"] = jrc_map["eic_p"] + + for col in [ + "Technology", "Set", "Efficiency", "DateIn", "DateRetrofit", + "DateOut", "Duration", "Volume_Mm3", "DamHeight_m", + "StorageCapacity_MWh", + ]: + df[col] = None + + df = df[df["Capacity"].notna() & (df["Capacity"] > 0)] + + return ( + df.pipe(clean_name) + .pipe(set_column_name, "JRC-PPDB-OPEN") + .pipe(config_filter, config) + ) + + @deprecated( deprecated_in="0.5.0", details="Use the JRC data instead", diff --git a/powerplantmatching/package_data/config.yaml b/powerplantmatching/package_data/config.yaml index aced4f37..e8993c9f 100644 --- a/powerplantmatching/package_data/config.yaml +++ b/powerplantmatching/package_data/config.yaml @@ -24,6 +24,7 @@ matching_sources: # wind in germany is provided by MASTR, nuclear is not block-wise, other filters are due to large deviations to other datasets - GPD: Capacity >= 1 and not (Country == 'Germany' and Fueltype == 'Wind') and not (Country in ['Czechia', 'Bulgaria', 'Romania'] and Fueltype == 'Hard Coal') and Fueltype != 'Nuclear' - JRC: Capacity >= 1 + - JRC-PPDB-OPEN: Capacity >= 1 # wind in germany is provided by MASTR, other filters are due to large deviations to other datasets - OPSD: not (Country == 'Germany' and Fueltype == 'Wind') and ((Capacity >= 1 and Fueltype != 'Solar') or Capacity >= 3) and not (Country == 'Spain' and Fueltype == 'Hard Coal') and not (Country == 'Italy' and Fueltype == 'Natural Gas') - BEYONDCOAL @@ -48,6 +49,7 @@ fully_included_sources: - BEYONDCOAL # include this selection of countries as they have poorer coverage in all other datasets - JRC: Country in ['Italy', 'Croatia', 'Serbia', 'Slovakia'] + - JRC-PPDB-OPEN # these sources skip unit aggregation for fully_included_sources not covered in matching_sources aggregate_only_matching_sources: @@ -96,6 +98,10 @@ JRC: reliability_score: 5 fn: jrc-hydro-power-plant-database.csv url: https://raw.githubusercontent.com/energy-modelling-toolkit/hydro-power-database/27e80f/data/jrc-hydro-power-plant-database.csv +JRC-PPDB-OPEN: + reliability_score: 5 + fn: JRC-PPDB-OPEN.ver1.0.zip + url: https://zenodo.org/records/3574566/files/JRC-PPDB-OPEN.ver1.0.zip GEO: net_capacity: false reliability_score: 2 From ab45a236e7c795a2578dd9b1067165de3d94014f Mon Sep 17 00:00:00 2001 From: Stan Buren Date: Sat, 11 Jul 2026 23:06:25 +0300 Subject: [PATCH 3/6] =?UTF-8?q?refactor:=20remove=20EIC=20pre-join=20?= =?UTF-8?q?=E2=80=94=20defer=20to=20PR=20#289?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #289 (MaykThewessen) implements a more robust EIC-first matching with degree-1 uniqueness checks that handles Alpine hydro schemes correctly. Our eic_codes.py now provides only validation utilities (is_valid_eic, extract_eics) and the EIC_PATTERN constant. The matching.py country_link reverts to the original Duke-only path. EIC-based matching will be handled by _match_by_eic from #289. --- powerplantmatching/eic_codes.py | 100 ++++++++++++++------------------ powerplantmatching/matching.py | 26 ++------- 2 files changed, 46 insertions(+), 80 deletions(-) diff --git a/powerplantmatching/eic_codes.py b/powerplantmatching/eic_codes.py index 6c177086..2edf5385 100644 --- a/powerplantmatching/eic_codes.py +++ b/powerplantmatching/eic_codes.py @@ -1,63 +1,47 @@ +"""EIC (Energy Identification Code) utilities. + +EIC codes are 16-character identifiers assigned by ENTSO-E. +The third character denotes the object type: 'W' = Resource Object +(power plants and generation units). +""" + import re + import pandas as pd -def eic_pre_join( - df_a: pd.DataFrame, - df_b: pd.DataFrame, - label_a: str, - label_b: str -) -> tuple[pd.DataFrame | None, pd.DataFrame, pd.DataFrame]: - """Find guaranteed matches via shared EIC codes. - - If two records share a valid EIC (16 chars, 3rd char = 'W'), - they are guaranteed to be the same plant. Returns: - - links_df: DataFrame with matched indices - - remaining_a: unmatched slice of df_a - - remaining_b: unmatched slice of df_b +# ENTSO-E EIC code pattern: 16 chars, 3rd char = 'W' (Resource Object) +EIC_PATTERN = re.compile(r"^..W.{13}$") + + +def is_valid_eic(code: str | None) -> bool: + """Return True if *code* is a valid EIC code.""" + if code is None: + return False + return bool(EIC_PATTERN.match(str(code))) + + +def extract_eics(series: pd.Series) -> dict[str, list]: + """Extract valid EIC codes from a pandas Series. + + Handles scalar strings and sets (as stored after aggregation). + + Returns + ------- + dict + ``{eic_code: [index, ...]}`` — each valid EIC mapped to the + list of row indices where it appears. """ - if "EIC" not in df_a.columns or "EIC" not in df_b.columns: - return None, df_a, df_b - - eic_pattern = re.compile(r"^..W.{13}$") - - def get_valid_eics(series): - result = {} - for idx, val in series.items(): - try: - if pd.isna(val): - continue - except (ValueError, TypeError): - pass - if isinstance(val, set): - for v in val: - if isinstance(v, str) and eic_pattern.match(v): - result.setdefault(v, []).append(idx) - elif isinstance(val, str) and eic_pattern.match(val): - result.setdefault(val, []).append(idx) - return result - - eics_a = get_valid_eics(df_a["EIC"]) - eics_b = get_valid_eics(df_b["EIC"]) - - shared = set(eics_a.keys()) & set(eics_b.keys()) - if not shared: - return None, df_a, df_b - - eic_links = [] - matched_a, matched_b = set(), set() - for eic in shared: - for ia in eics_a[eic]: - if ia in matched_a: + result: dict[str, list] = {} + for idx, val in series.items(): + try: + if pd.isna(val): continue - for ib in eics_b[eic]: - if ib in matched_b: - continue - eic_links.append({label_a: ia, label_b: ib}) - matched_a.add(ia) - matched_b.add(ib) - break - - remaining_a = df_a.drop(index=list(matched_a), errors="ignore") - remaining_b = df_b.drop(index=list(matched_b), errors="ignore") - links_df = pd.DataFrame(eic_links, columns=[label_a, label_b]) - return links_df, remaining_a, remaining_b + except (ValueError, TypeError): + pass + if isinstance(val, set): + for v in val: + if isinstance(v, str) and EIC_PATTERN.match(v): + result.setdefault(v, []).append(idx) + elif isinstance(val, str) and EIC_PATTERN.match(val): + result.setdefault(val, []).append(idx) + return result diff --git a/powerplantmatching/matching.py b/powerplantmatching/matching.py index 03c90d6d..6fcfeaaf 100644 --- a/powerplantmatching/matching.py +++ b/powerplantmatching/matching.py @@ -81,28 +81,10 @@ def country_link(dfs, country): # country_selector for both dataframes sel_country_b = [df["Country"] == country for df in dfs] # only append if country appears in both dataframes - if not all(sel.any() for sel in sel_country_b): - return pd.DataFrame(columns=[*labels, "scores"]) - - df_a = dfs[0][sel_country_b[0]] - df_b = dfs[1][sel_country_b[1]] - - # EIC pre-join: deterministic matches via shared EIC codes - from powerplantmatching.eic_codes import eic_pre_join - eic_matches, df_a, df_b = eic_pre_join(df_a, df_b, labels[0], labels[1]) - - # Duke on remaining unmatched records - if len(df_a) > 0 and len(df_b) > 0: - duke_links = duke([df_a, df_b], labels, **dukeargs) - if eic_matches is not None and not eic_matches.empty: - eic_matches["scores"] = 1.0 - if not duke_links.empty: - return pd.concat([eic_matches, duke_links], ignore_index=True) - return eic_matches - return duke_links - elif eic_matches is not None and not eic_matches.empty: - eic_matches["scores"] = 1.0 - return eic_matches + if all(sel.any() for sel in sel_country_b): + return duke( + [df[sel] for df, sel in zip(dfs, sel_country_b)], labels, **dukeargs + ) else: return pd.DataFrame(columns=[*labels, "scores"]) From e0d82698fb3c49f6682143b9da953967ef56ff03 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 11 Jul 2026 20:10:27 +0000 Subject: [PATCH 4/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- powerplantmatching/cleaning.py | 4 +++- powerplantmatching/data.py | 30 ++++++++++++++++++++---------- powerplantmatching/matching.py | 3 ++- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/powerplantmatching/cleaning.py b/powerplantmatching/cleaning.py index 46041c9c..cc96df11 100644 --- a/powerplantmatching/cleaning.py +++ b/powerplantmatching/cleaning.py @@ -524,7 +524,9 @@ def aggregate_units( df = ( df.assign( - **df[weighted_cols].div(df["Capacity"].replace(0, pd.NA), axis=0).where(lambda df: df != 0) + **df[weighted_cols] + .div(df["Capacity"].replace(0, pd.NA), axis=0) + .where(lambda df: df != 0) ) .reset_index(drop=True) .pipe(clean_name) diff --git a/powerplantmatching/data.py b/powerplantmatching/data.py index 51da7ffc..a7053e46 100644 --- a/powerplantmatching/data.py +++ b/powerplantmatching/data.py @@ -508,6 +508,7 @@ def JRC_PPDB_OPEN(raw=False, update=False, config=None): fn = get_raw_file("JRC-PPDB-OPEN", update, config) from zipfile import ZipFile + with ZipFile(fn, "r") as zf: with zf.open("JRC_OPEN_UNITS.csv") as f: jrc = pd.read_csv(f) @@ -520,14 +521,16 @@ def JRC_PPDB_OPEN(raw=False, update=False, config=None): # Aggregate generation units to production units jrc_map = ( jrc.groupby("eic_p") - .agg({ - "lat": "mean", - "lon": "mean", - "name_p": "first", - "capacity_p": "sum", - "type_g": "first", - "country": "first", - }) + .agg( + { + "lat": "mean", + "lon": "mean", + "name_p": "first", + "capacity_p": "sum", + "type_g": "first", + "country": "first", + } + ) .reset_index() ) @@ -542,8 +545,15 @@ def JRC_PPDB_OPEN(raw=False, update=False, config=None): df["projectID"] = jrc_map["eic_p"] for col in [ - "Technology", "Set", "Efficiency", "DateIn", "DateRetrofit", - "DateOut", "Duration", "Volume_Mm3", "DamHeight_m", + "Technology", + "Set", + "Efficiency", + "DateIn", + "DateRetrofit", + "DateOut", + "Duration", + "Volume_Mm3", + "DamHeight_m", "StorageCapacity_MWh", ]: df[col] = None diff --git a/powerplantmatching/matching.py b/powerplantmatching/matching.py index 6fcfeaaf..ca0ae0ad 100644 --- a/powerplantmatching/matching.py +++ b/powerplantmatching/matching.py @@ -288,7 +288,8 @@ def reduce_matched_dataframe(df, show_orig_names=False, config=None): "DateOut": "max", "projectID": lambda x: dict(x.droplevel(0).dropna()), "EIC": lambda x: set( - v for val in x.dropna() + v + for val in x.dropna() for v in (val if isinstance(val, set) else [val]) if isinstance(v, str) ), From 56bc1d8409c2d734c4a16e3f9ef5d7dba4dbc2e8 Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 7 Oct 2026 10:14:25 +0200 Subject: [PATCH 5/6] fix JRC-PPDB-OPEN integration as matching source - map fuel types, sum generation unit capacities, filter by status - rename source to JRC_PPDB_OPEN so collect() can resolve the importer - use as matching source only (reliability 2), not fully included - remove unused eic_codes.py and revert no-op change in aggregate_units --- docs/release-notes.md | 2 +- powerplantmatching/cleaning.py | 4 +- powerplantmatching/data.py | 83 +++++++-------------- powerplantmatching/eic_codes.py | 47 ------------ powerplantmatching/package_data/config.yaml | 8 +- 5 files changed, 32 insertions(+), 112 deletions(-) delete mode 100644 powerplantmatching/eic_codes.py diff --git a/docs/release-notes.md b/docs/release-notes.md index 1f1f48a0..b52dedf3 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -8,7 +8,7 @@ SPDX-License-Identifier: MIT ## Upcoming Version -* +* Add the [JRC Open Power Plants Database](https://doi.org/10.5281/zenodo.3574566) (`JRC_PPDB_OPEN`) as a matching source. It provides coordinates for ENTSO-E production units, which helps to match ENTSOE with other datasets. ([PR #306](https://github.com/PyPSA/powerplantmatching/pull/306)) ## [v0.9.0](https://github.com/PyPSA/powerplantmatching/releases/tag/v0.9.0) (3rd October 2026) diff --git a/powerplantmatching/cleaning.py b/powerplantmatching/cleaning.py index cc96df11..fbe80c74 100644 --- a/powerplantmatching/cleaning.py +++ b/powerplantmatching/cleaning.py @@ -524,9 +524,7 @@ def aggregate_units( df = ( df.assign( - **df[weighted_cols] - .div(df["Capacity"].replace(0, pd.NA), axis=0) - .where(lambda df: df != 0) + **df[weighted_cols].div(df["Capacity"], axis=0).where(lambda df: df != 0) ) .reset_index(drop=True) .pipe(clean_name) diff --git a/powerplantmatching/data.py b/powerplantmatching/data.py index 36af2383..688e35e3 100644 --- a/powerplantmatching/data.py +++ b/powerplantmatching/data.py @@ -486,14 +486,12 @@ def set_large_spanish_stores_to_reservoirs(df): def JRC_PPDB_OPEN(raw=False, update=False, config=None): """ - Importer for the JRC Open Power Plants Database (JRC-PPDB-OPEN). + Importer for the JRC Open Power Plants Database (JRC-PPDB-OPEN, + Kanellopoulos et al., 2019, doi:10.5281/zenodo.3574566). - Published by the European Commission's Joint Research Centre - (DOI: 10.5281/zenodo.3574566), this database was created - specifically to link ENTSO-E EIC codes with geographic - coordinates. It covers ~70% of large European power plants - and provides a deterministic bridge between EIC-based - operational data (ENTSO-E) and spatial data (OSM/GEM/GEO). + The database extends the ENTSO-E production units with coordinates. + Generation units are aggregated to production units, which have the + same EIC codes as the ENTSOE data. Parameters ---------- @@ -506,64 +504,35 @@ def JRC_PPDB_OPEN(raw=False, update=False, config=None): """ config = get_config() if config is None else config - fn = get_raw_file("JRC-PPDB-OPEN", update, config) + fn = get_raw_file("JRC_PPDB_OPEN", update, config) - from zipfile import ZipFile - - with ZipFile(fn, "r") as zf: - with zf.open("JRC_OPEN_UNITS.csv") as f: - jrc = pd.read_csv(f) + with ZipFile(fn) as zf, zf.open("JRC_OPEN_UNITS.csv") as f: + df = pd.read_csv(f) if raw: - return jrc + return df - jrc = jrc[jrc["eic_p"].notna() & jrc["lat"].notna() & jrc["lon"].notna()] + status_list = config["JRC_PPDB_OPEN"].get("status", ["COMMISSIONED"]) # noqa: F841 - # Aggregate generation units to production units - jrc_map = ( - jrc.groupby("eic_p") + return ( + df.assign(status_g=df.status_g.str.upper()) + .query("status_g in @status_list") + .groupby("eic_p") .agg( - { - "lat": "mean", - "lon": "mean", - "name_p": "first", - "capacity_p": "sum", - "type_g": "first", - "country": "first", - } + Name=("name_p", "first"), + Fueltype=("type_g", "first"), + Country=("country", "first"), + Capacity=("capacity_g", "sum"), + DateIn=("year_commissioned", "min"), + lat=("lat", "mean"), + lon=("lon", "mean"), ) + .rename_axis("projectID") .reset_index() - ) - - df = pd.DataFrame() - df["Name"] = jrc_map["name_p"] - df["Fueltype"] = jrc_map["type_g"] - df["Country"] = jrc_map["country"] - df["Capacity"] = jrc_map["capacity_p"] - df["lat"] = jrc_map["lat"] - df["lon"] = jrc_map["lon"] - df["EIC"] = jrc_map["eic_p"] - df["projectID"] = jrc_map["eic_p"] - - for col in [ - "Technology", - "Set", - "Efficiency", - "DateIn", - "DateRetrofit", - "DateOut", - "Duration", - "Volume_Mm3", - "DamHeight_m", - "StorageCapacity_MWh", - ]: - df[col] = None - - df = df[df["Capacity"].notna() & (df["Capacity"] > 0)] - - return ( - df.pipe(clean_name) - .pipe(set_column_name, "JRC-PPDB-OPEN") + .assign(EIC=lambda df: df.projectID, Technology=np.nan, Set=np.nan) + .pipe(gather_specifications, config=config) + .pipe(clean_name) + .pipe(set_column_name, "JRC_PPDB_OPEN") .pipe(config_filter, config) ) diff --git a/powerplantmatching/eic_codes.py b/powerplantmatching/eic_codes.py deleted file mode 100644 index 2edf5385..00000000 --- a/powerplantmatching/eic_codes.py +++ /dev/null @@ -1,47 +0,0 @@ -"""EIC (Energy Identification Code) utilities. - -EIC codes are 16-character identifiers assigned by ENTSO-E. -The third character denotes the object type: 'W' = Resource Object -(power plants and generation units). -""" - -import re - -import pandas as pd - -# ENTSO-E EIC code pattern: 16 chars, 3rd char = 'W' (Resource Object) -EIC_PATTERN = re.compile(r"^..W.{13}$") - - -def is_valid_eic(code: str | None) -> bool: - """Return True if *code* is a valid EIC code.""" - if code is None: - return False - return bool(EIC_PATTERN.match(str(code))) - - -def extract_eics(series: pd.Series) -> dict[str, list]: - """Extract valid EIC codes from a pandas Series. - - Handles scalar strings and sets (as stored after aggregation). - - Returns - ------- - dict - ``{eic_code: [index, ...]}`` — each valid EIC mapped to the - list of row indices where it appears. - """ - result: dict[str, list] = {} - for idx, val in series.items(): - try: - if pd.isna(val): - continue - except (ValueError, TypeError): - pass - if isinstance(val, set): - for v in val: - if isinstance(v, str) and EIC_PATTERN.match(v): - result.setdefault(v, []).append(idx) - elif isinstance(val, str) and EIC_PATTERN.match(val): - result.setdefault(val, []).append(idx) - return result diff --git a/powerplantmatching/package_data/config.yaml b/powerplantmatching/package_data/config.yaml index a23bc216..f4d63598 100644 --- a/powerplantmatching/package_data/config.yaml +++ b/powerplantmatching/package_data/config.yaml @@ -24,7 +24,7 @@ matching_sources: # wind in germany is provided by MASTR, nuclear is not block-wise, other filters are due to large deviations to other datasets - GPD: Capacity >= 1 and not (Country == 'Germany' and Fueltype == 'Wind') and not (Country in ['Czechia', 'Bulgaria', 'Romania'] and Fueltype == 'Hard Coal') and Fueltype != 'Nuclear' - JRC: Capacity >= 1 - - JRC-PPDB-OPEN: Capacity >= 1 + - JRC_PPDB_OPEN: not (Country == 'Germany' and Fueltype == 'Wind') # wind in germany is provided by MASTR, other filters are due to large deviations to other datasets - OPSD: not (Country == 'Germany' and Fueltype == 'Wind') and ((Capacity >= 1 and Fueltype != 'Solar') or Capacity >= 3) and not (Country == 'Spain' and Fueltype == 'Hard Coal') and not (Country == 'Italy' and Fueltype == 'Natural Gas') - BEYONDCOAL @@ -50,7 +50,6 @@ fully_included_sources: - BEYONDCOAL # include this selection of countries as they have poorer coverage in all other datasets - JRC: Country in ['Italy', 'Croatia', 'Serbia', 'Slovakia'] - - JRC-PPDB-OPEN # these sources skip unit aggregation for fully_included_sources not covered in matching_sources aggregate_only_matching_sources: @@ -99,8 +98,9 @@ JRC: reliability_score: 5 fn: jrc-hydro-power-plant-database.csv url: https://raw.githubusercontent.com/energy-modelling-toolkit/hydro-power-database/3e8a837/data/jrc-hydro-power-plant-database.csv -JRC-PPDB-OPEN: - reliability_score: 5 +JRC_PPDB_OPEN: + reliability_score: 2 + status: ["COMMISSIONED", "RESERVE", "MOTHBALLED", "CONSTRUCTION"] fn: JRC-PPDB-OPEN.ver1.0.zip url: https://zenodo.org/records/3574566/files/JRC-PPDB-OPEN.ver1.0.zip GEO: From e0a292c1d3e208375e7193cf8a19ccc99343253b Mon Sep 17 00:00:00 2001 From: Fabian Neumann Date: Wed, 7 Oct 2026 11:05:47 +0200 Subject: [PATCH 6/6] use JRC-PPDB-OPEN coordinates in ENTSOE instead of as matching source - fill ENTSOE coordinates from JRC-PPDB-OPEN via production unit EIC codes - use ENTSOE coordinates in reduced output only as fallback - remove JRC_PPDB_OPEN from matching sources to avoid ENTSO-E confirming itself --- docs/release-notes.md | 2 +- powerplantmatching/data.py | 14 +++++----- powerplantmatching/matching.py | 6 +++++ powerplantmatching/package_data/config.yaml | 2 -- test/test_data.py | 4 +++ test/test_matching.py | 29 +++++++++++++++++++++ 6 files changed, 48 insertions(+), 9 deletions(-) create mode 100644 test/test_matching.py diff --git a/docs/release-notes.md b/docs/release-notes.md index b52dedf3..27b0f935 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -8,7 +8,7 @@ SPDX-License-Identifier: MIT ## Upcoming Version -* Add the [JRC Open Power Plants Database](https://doi.org/10.5281/zenodo.3574566) (`JRC_PPDB_OPEN`) as a matching source. It provides coordinates for ENTSO-E production units, which helps to match ENTSOE with other datasets. ([PR #306](https://github.com/PyPSA/powerplantmatching/pull/306)) +* Add the [JRC Open Power Plants Database](https://doi.org/10.5281/zenodo.3574566) (`pm.data.JRC_PPDB_OPEN()`). Its coordinates fill the ENTSOE data via EIC codes, which helps to match ENTSOE with other datasets. In the matched output, other sources' coordinates take priority over these. ([PR #306](https://github.com/PyPSA/powerplantmatching/pull/306)) ## [v0.9.0](https://github.com/PyPSA/powerplantmatching/releases/tag/v0.9.0) (3rd October 2026) diff --git a/powerplantmatching/data.py b/powerplantmatching/data.py index 688e35e3..205ef99c 100644 --- a/powerplantmatching/data.py +++ b/powerplantmatching/data.py @@ -489,9 +489,10 @@ def JRC_PPDB_OPEN(raw=False, update=False, config=None): Importer for the JRC Open Power Plants Database (JRC-PPDB-OPEN, Kanellopoulos et al., 2019, doi:10.5281/zenodo.3574566). - The database extends the ENTSO-E production units with coordinates. - Generation units are aggregated to production units, which have the - same EIC codes as the ENTSOE data. + The database extends the ENTSO-E production units with coordinates, + which are used to fill the coordinates of the ENTSOE data. Generation + units are aggregated to production units, which have the same EIC codes + as the ENTSOE data. Parameters ---------- @@ -741,7 +742,7 @@ def ENTSOE( ): """ Importer for the list of installed generators provided by the ENTSO-E - Transparency Project. Geographical information is not given. + Transparency Project. Coordinates are taken from JRC-PPDB-OPEN via EIC codes. If update=True, the dataset is parsed through a request to 'https://transparency.entsoe.eu/generation/r2/\ installedCapacityPerProductionUnit/show', @@ -831,6 +832,7 @@ def retrieve_data(token): fn = _package_data("entsoe_country_codes.csv") COUNTRY_MAP = pd.read_csv(fn, index_col=0).rename(index=str).Country + coords = JRC_PPDB_OPEN(config=config).set_index("projectID") return ( df.rename_axis(index="projectID") @@ -844,8 +846,8 @@ def retrieve_data(token): Capacity=lambda df: pd.to_numeric(df.Capacity), Technology=np.nan, Set=np.nan, - lat=np.nan, - lon=np.nan, + lat=lambda df: df.projectID.map(coords.lat), + lon=lambda df: df.projectID.map(coords.lon), ) .powerplant.convert_alpha2_to_country() # .pipe(fill_geoposition, **fill_geoposition_kwargs) diff --git a/powerplantmatching/matching.py b/powerplantmatching/matching.py index e1933d9f..7855e884 100644 --- a/powerplantmatching/matching.py +++ b/powerplantmatching/matching.py @@ -297,6 +297,12 @@ def reduce_matched_dataframe(df, show_orig_names=False, config=None): ) props_for_groups = pd.Series(props_for_groups)[cols].to_dict() + # ENTSOE coordinates are filled from JRC-PPDB-OPEN, use them only as fallback + if "ENTSOE" in sources: + df = df.copy() + others = df.lat.drop(columns="ENTSOE").notna().any(axis=1) + df.loc[others, [("lat", "ENTSOE"), ("lon", "ENTSOE")]] = np.nan + # set low priority on Fueltype 'Other' and Set 'PP' # turn it since aggregating only possible for axis=0 sdf = ( diff --git a/powerplantmatching/package_data/config.yaml b/powerplantmatching/package_data/config.yaml index f4d63598..9a5ba94e 100644 --- a/powerplantmatching/package_data/config.yaml +++ b/powerplantmatching/package_data/config.yaml @@ -24,7 +24,6 @@ matching_sources: # wind in germany is provided by MASTR, nuclear is not block-wise, other filters are due to large deviations to other datasets - GPD: Capacity >= 1 and not (Country == 'Germany' and Fueltype == 'Wind') and not (Country in ['Czechia', 'Bulgaria', 'Romania'] and Fueltype == 'Hard Coal') and Fueltype != 'Nuclear' - JRC: Capacity >= 1 - - JRC_PPDB_OPEN: not (Country == 'Germany' and Fueltype == 'Wind') # wind in germany is provided by MASTR, other filters are due to large deviations to other datasets - OPSD: not (Country == 'Germany' and Fueltype == 'Wind') and ((Capacity >= 1 and Fueltype != 'Solar') or Capacity >= 3) and not (Country == 'Spain' and Fueltype == 'Hard Coal') and not (Country == 'Italy' and Fueltype == 'Natural Gas') - BEYONDCOAL @@ -99,7 +98,6 @@ JRC: fn: jrc-hydro-power-plant-database.csv url: https://raw.githubusercontent.com/energy-modelling-toolkit/hydro-power-database/3e8a837/data/jrc-hydro-power-plant-database.csv JRC_PPDB_OPEN: - reliability_score: 2 status: ["COMMISSIONED", "RESERVE", "MOTHBALLED", "CONSTRUCTION"] fn: JRC-PPDB-OPEN.ver1.0.zip url: https://zenodo.org/records/3574566/files/JRC-PPDB-OPEN.ver1.0.zip diff --git a/test/test_data.py b/test/test_data.py index d3614371..669e6024 100755 --- a/test/test_data.py +++ b/test/test_data.py @@ -36,6 +36,10 @@ def test_data_request_processed(source): assert df.columns.to_list() == config["target_columns"] +def test_ENTSOE_coordinates(): + assert data.ENTSOE().lat.notna().mean() > 0.5 + + def test_OPSD_VRE(): df = pm.data.OPSD_VRE() assert not df.empty diff --git a/test/test_matching.py b/test/test_matching.py new file mode 100644 index 00000000..8669b895 --- /dev/null +++ b/test/test_matching.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Contributors to powerplantmatching +# +# SPDX-License-Identifier: MIT + +import numpy as np +import pandas as pd +import pytest + +import powerplantmatching as pm +from powerplantmatching.matching import reduce_matched_dataframe + + +@pytest.mark.parametrize("gpd_lat, expected", [(50.0, 50.0), (np.nan, 49.0)]) +def test_entsoe_coordinates_are_fallback(gpd_lat, expected): + plant = {"Name": "Plant", "Fueltype": "Hydro", "Country": "Austria", "lon": 10.0} + df = pd.concat( + { + "ENTSOE": pd.DataFrame([{**plant, "Capacity": 100.0, "lat": 49.0}]), + "GPD": pd.DataFrame([{**plant, "Capacity": 90.0, "lat": gpd_lat}]), + }, + axis=1, + ).swaplevel(axis=1) + cols = pd.MultiIndex.from_product( + [pm.get_config()["target_columns"], ["ENTSOE", "GPD"]] + ) + df = df.reindex(columns=cols) + reduced = reduce_matched_dataframe(df) + assert reduced.loc[0, "lat"] == expected + assert reduced.loc[0, "Capacity"] == 100.0