diff --git a/docs/basics.md b/docs/basics.md index 8c1ebdd2..c7df1762 100644 --- a/docs/basics.md +++ b/docs/basics.md @@ -12,6 +12,7 @@ The dataset combines the data of all the data sources listed in [Data-Sources](# - **Fueltype** - {Solid Biomass, Biogas, Geothermal, Hard Coal, Hydro, Lignite, Nuclear, Natural Gas, Oil, Solar, Wind, Other} - **Technology** - {CCGT, OCGT, Steam Turbine, Combustion Engine, Run-Of-River, Pumped Storage, Reservoir} - **Set** - {Power Plant (PP), Combined Heat and Power (CHP), Storages (Stores)} +- **Status** - life-cycle stage {announced, construction, operating, mothballed, retired}. If the data sources do not agree, the latest stage is used. - **Capacity** - [MW] - **Duration** - Maximum state of charge capacity in terms of hours at full output capacity - **Dam Information** - Dam volume [Mm^3] and Dam Height [m] diff --git a/docs/custom-config.md b/docs/custom-config.md index c6fe48ac..ab11b9db 100644 --- a/docs/custom-config.md +++ b/docs/custom-config.md @@ -16,6 +16,7 @@ and change the `.powerplantmaching_config.yaml` file according to your wishes. T - determine the global set of **countries** and **fueltypes** - determine which data sources to combine and which data sources should completely be contained in the final dataset +- select the life-cycle stages of the power plants with the key `status`, for all data sources and optionally per data source (see `target_status` for the stages and the mapping of the status labels of each data source) - individually filter data sources via [`pandas.DataFrame.query`](http://pandas.pydata.org/pandas-docs/stable/indexing.html#the-query-method) statements set as an argument of data source name. See the default [config.yaml file](https://github.com/PyPSA/powerplantmatching/blob/master/powerplantmatching/package_data/config.yaml) as an example Optionally you can: diff --git a/docs/release-notes.md b/docs/release-notes.md index 1f1f48a0..2cae572d 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -8,7 +8,16 @@ SPDX-License-Identifier: MIT ## Upcoming Version -* +* New column `Status` with the life-cycle stage of the power plant: `announced`, `construction`, `operating`, `mothballed` or `retired`. ([#322](https://github.com/PyPSA/powerplantmatching/issues/322)) + * **Important:** the output now also includes announced, mothballed and retired power plants. Announced projects add several hundred GW, and most of them have no `DateIn`. Filter with the `Status` column, e.g. `df.query("Status in ['operating', 'construction']")`; a filter on `DateIn` and `DateOut` alone is not sufficient. + * The status labels of the data sources are mapped to the stages in the new config section `target_status`. The new key `status` selects the stages to keep for all data sources; by default, all stages are kept. The key `status` of a data source overrides it and now selects stages and not the status labels of the data source. + * Data sources without status data list existing power plants and get the status `operating` (ENTSOE, GPD, JRC, OSM, GHR and OPSD outside Germany). GEO gets the status `retired` if `DateOut` is before the current year. The status of OPSD for Germany is not used, as it is outdated. + * If matched data sources do not agree, the latest stage is used. Units with different stages are not aggregated into one power plant. + * Announced projects and projects under construction are not matched with other data sources, because they often have the name and site of existing power plants. They are added without matching from the fully included data sources. + * BEYONDCOAL is no longer a fully included data source, as GEM includes the same coal units with a higher reliability. This prevents double counting of units with a different status in the two data sources. Known limitation: if two matched data sources group the units of a power plant differently by status, a unit can still be counted twice (e.g. the retired unit 9 of Walsum in Germany, from MASTR and GEM). + * The `DateIn` and `DateOut` values that were estimated from the status are removed (mothballed plants in the GEM coal and oil and gas trackers, temporarily shut down units in MASTR, nuclear plants under construction in WIKIPEDIA). +* Bugfix in `aggregate_units()`: nuclear units are not aggregated within a data source, as set by `fueltypes_with_blocks` in the config section `clean_name`. Before, a wrong config key disabled this setting. +* Bugfix in `pm.data.GHR()`: read capacity, coordinates and technology from the correct columns, and read the commissioning year without conversion to a date (all values were 1970 before). ## [v0.9.0](https://github.com/PyPSA/powerplantmatching/releases/tag/v0.9.0) (3rd October 2026) diff --git a/powerplantmatching/cleaning.py b/powerplantmatching/cleaning.py index fbe80c74..0378110a 100644 --- a/powerplantmatching/cleaning.py +++ b/powerplantmatching/cleaning.py @@ -34,6 +34,7 @@ def mode(x): "Fueltype": mode, "Technology": mode, "Set": mode, + "Status": mode, "Country": mode, "Capacity": "sum", "lat": "mean", @@ -345,6 +346,29 @@ def gather_set_info(df, search_col=["Name", "Fueltype", "Technology"], config=No return df.assign(Set=Set) +def map_status(df, config=None): + """ + Map the raw labels in column "Status" to the life-cycle stages given by + the `config` under the section `target_status`. Entries with other labels + (e.g. cancelled) are removed. + + Parameters + ---------- + df : pandas.DataFrame + DataFrame with raw labels in column "Status". + config : dict, default None + Custom configuration, defaults to + `powerplantmatching.config.get_config()`. + """ + if config is None: + config = get_config() + + stages = config["target_status"] + labels = {label: stage for stage, labels in stages.items() for label in labels} + status = df.Status.str.lower().map(labels) + return df.assign(Status=status)[status.notna()] + + # @deprecated( # deprecated_in="0.5", # ) @@ -476,7 +500,9 @@ def aggregate_units( cols = config["target_columns"] weighted_cols = list({"Efficiency", "Duration"} & set(cols)) - str_cols = list({"Name", "Country", "Fueltype", "Technology", "Set"} & set(cols)) + str_cols = list( + {"Name", "Country", "Fueltype", "Technology", "Set", "Status"} & set(cols) + ) props_for_groups = {k: v for k, v in AGGREGATION_FUNCTIONS.items() if k in cols} df = df.assign( @@ -497,8 +523,8 @@ def aggregate_units( break block_query = None - if with_blocks := config["clean_name"].get("fuel_type_with_blocks", []): # noqa - block_query = "Fueltype in @with_blocks" + if with_blocks := config["clean_name"].get("fueltypes_with_blocks", []): # noqa + block_query = "Fueltype not in @with_blocks" if country_wise: countries = df.Country.unique() @@ -512,7 +538,9 @@ def aggregate_units( duplicates = duke(df.query(query) if query else df, threads=threads) df = cliques(df, duplicates) - df = df.groupby("grouped").agg(props_for_groups) + # do not merge units at different life-cycle stages + by = ["grouped", "Status"] if "Status" in cols else ["grouped"] + df = df.groupby([df[c] for c in by]).agg(props_for_groups) no_downcast_ctx = ( contextlib.nullcontext() diff --git a/powerplantmatching/collection.py b/powerplantmatching/collection.py index a677ae52..67d1fb5b 100644 --- a/powerplantmatching/collection.py +++ b/powerplantmatching/collection.py @@ -243,14 +243,12 @@ def powerplants( else: matched = matched[matched.lat.notnull()] + # keep units of one power plant at different life-cycle stages + subset = [c for c in ["Name", "Fueltype", "Country", "Status"] if c in matched] if isinstance(matched.columns, pd.MultiIndex): - matched = ( - matched.stack(future_stack=True) - .drop_duplicates(["Name", "Fueltype", "Country"]) - .unstack(-1) - ) + matched = matched.stack(future_stack=True).drop_duplicates(subset).unstack(-1) else: - matched = matched.drop_duplicates(["Name", "Fueltype", "Country"]) + matched = matched.drop_duplicates(subset) matched.reset_index(drop=True).to_csv(fn, index_label="id", encoding="utf-8") diff --git a/powerplantmatching/data.py b/powerplantmatching/data.py index 5dfba6f0..1f6702b7 100644 --- a/powerplantmatching/data.py +++ b/powerplantmatching/data.py @@ -25,6 +25,7 @@ gather_fueltype_info, gather_set_info, gather_specifications, + map_status, ) from .core import PANDAS_V3, _package_data, get_config from .heuristics import PLZ_to_LatLon_map, scale_to_net_capacities @@ -78,13 +79,11 @@ def BEYONDCOAL(raw=False, update=False, config=None): if raw: return df - status_list = config["BEYONDCOAL"].get("status", ["operational"]) # noqa - RENAME_COLUMNS = { "Unit name": "Name", "Fuel type": "Fueltype", "Commissioning year": "DateIn", - "Unit status\n(detailed)": "status", + "Unit status\n(detailed)": "Status", "BFF unit ID": "projectID", } @@ -111,7 +110,7 @@ def BEYONDCOAL(raw=False, update=False, config=None): df_final = ( df.rename(columns=RENAME_COLUMNS) - .query("status in @status_list") + .pipe(map_status, config=config) .assign( DateOut=date_out, projectID=lambda df: "BEYOND-" + df.projectID, @@ -194,6 +193,7 @@ def OPSD( opsd_EU.rename(columns=str.title) .rename(columns=EU_RENAME_COLUMNS) .eval("DateRetrofit = DateIn") + .assign(Status="operating") .query("Country != 'DE'") .query( "not Name.str.startswith('BNA')" @@ -224,7 +224,8 @@ def OPSD( if statusDE is not None: opsd_DE = opsd_DE.loc[opsd_DE.Status.isin(statusDE)] - opsd_DE = opsd_DE.reindex(columns=config["target_columns"]) + # the status is outdated + opsd_DE = opsd_DE.drop(columns="Status").reindex(columns=config["target_columns"]) return ( pd.concat([opsd_EU, opsd_DE], ignore_index=True) @@ -329,6 +330,10 @@ def to_year(ds): res = scale_to_net_capacities(res) res = convert_to_short_name(res) res = set_column_name(res, "GEO") + # GEO keeps decommissioned power plants + res["Status"] = np.where( + res.DateOut < pd.Timestamp.now().year, "retired", "operating" + ) res = config_filter(res, config) res["projectID"] = "GEO-" + res.projectID.astype(str) @@ -480,6 +485,7 @@ def set_large_spanish_stores_to_reservoirs(df): .powerplant.convert_alpha2_to_country() .pipe(clean_name) .pipe(set_large_spanish_stores_to_reservoirs) + .assign(Status="operating") .pipe(set_column_name, "JRC") .pipe(config_filter, config) ) @@ -622,6 +628,7 @@ def GPD(raw=False, update=False, config=None, filter_other_dbs=True): config=config, ) .pipe(clean_name) + .assign(Status="operating") .pipe(set_column_name, "GPD") .pipe(config_filter, config) .pipe(gather_specifications, config=config) @@ -670,9 +677,10 @@ def WIKIPEDIA(raw=False, update=False, config=None): Fueltype="Nuclear", Set="PP", projectID=lambda df: "WIKIPEDIA-" + df.index.astype(str), - # plants which are not yet built are set to 2027 - DateIn=lambda df: df.DateIn.where(~df.Status.str.contains("In Bau"), 2027), + # remove notes like "seit 2005" or "[12]" + Status=lambda df: df.Status.str.split(r"\s{2}|\s[(\[]").str[0], ) + .pipe(map_status, config=config) .pipe(clean_name) .pipe(set_column_name, "WIKIPEDIA") .pipe(config_filter, config) @@ -801,6 +809,7 @@ def retrieve_data(token): .query("Capacity > 0") .pipe(gather_specifications, config=config) .pipe(clean_name) + .assign(Status="operating") .pipe(set_column_name, "ENTSOE") .pipe(config_filter, config) ) @@ -898,6 +907,7 @@ def OSM(raw=False, update=False, config=None): config=config, ) .pipe(clean_name) + .assign(Status="operating") .pipe(set_column_name, "OSM") .pipe(config_filter, config) ) @@ -1758,8 +1768,6 @@ def GBPT(raw=False, update=False, config=None): "bioenergy: refuse (syngas)": "Solid Biomass", } - status_list = config["GBPT"].get("status", ["operating"]) # noqa: F841 - df = df.rename(columns=RENAME_COLUMNS) df_final = ( df.pipe(set_column_name, "GBPT") @@ -1774,7 +1782,7 @@ def GBPT(raw=False, update=False, config=None): lambda v: fueltype_dict[v.split(",")[0].strip()] ), ) - .query("Status in @status_list") + .pipe(map_status, config=config) .pipe(lambda x: x[df.columns.intersection(config.get("target_columns"))]) .assign(Technology=np.nan) .assign(Set=np.nan) @@ -1817,8 +1825,6 @@ def GNPT(raw=False, update=False, config=None): "GEM unit ID": "projectID", } - status_list = config["GNPT"].get("status", ["operating"]) # noqa: F841 - df = df.rename(columns=RENAME_COLUMNS) df_final = ( df.pipe(set_column_name, "GNPT") @@ -1834,7 +1840,7 @@ def GNPT(raw=False, update=False, config=None): lat=df["lat"].apply(pd.to_numeric, errors="coerce"), lon=df["lon"].apply(pd.to_numeric, errors="coerce"), ) - .query("Status in @status_list") + .pipe(map_status, config=config) .pipe(lambda x: x[df.columns.intersection(config.get("target_columns"))]) .assign(Fueltype="Nuclear") .assign(Technology="Steam Turbine") @@ -1902,14 +1908,6 @@ def GCPT(raw=False, update=False, config=None): planned_retirement = df["Planned retirement"].apply(pd.to_numeric, errors="coerce") - # conservative assumption that mothballed plants (without fixed retirement - # date) went out of operation in 2024 - mothballed_retirement = df["Status"].apply( - lambda x: 2024 if x == "mothballed" else np.nan - ) - - status_list = config["GCPT"].get("status", ["operating"]) # noqa: F841 - BTU_PER_KWH = 3412.14 df = df.rename(columns=RENAME_COLUMNS) @@ -1925,14 +1923,13 @@ def GCPT(raw=False, update=False, config=None): DateIn=df["DateIn"].apply(pd.to_numeric, errors="coerce"), DateOut=df["DateOut"] .apply(pd.to_numeric, errors="coerce") - .combine_first(planned_retirement) - .combine_first(mothballed_retirement), + .combine_first(planned_retirement), lat=df["lat"].apply(pd.to_numeric, errors="coerce"), lon=df["lon"].apply(pd.to_numeric, errors="coerce"), Set=df["CHP"].replace({"yes": "CHP", "no": "PP"}), Efficiency=BTU_PER_KWH / df["Heat rate (Btu per kWh)"], ) - .query("Status in @status_list") + .pipe(map_status, config=config) .pipe(lambda x: x[df.columns.intersection(config.get("target_columns"))]) .pipe( lambda x: x.replace( @@ -1979,8 +1976,6 @@ def GGTPT(raw=False, update=False, config=None): "GEM unit ID": "projectID", } - status_list = config["GGTPT"].get("status", ["operating"]) # noqa: F841 - df = df.rename(columns=RENAME_COLUMNS) df_final = ( df.pipe(set_column_name, "GGTPT") @@ -1992,7 +1987,7 @@ def GGTPT(raw=False, update=False, config=None): lat=df["lat"].apply(pd.to_numeric, errors="coerce"), lon=df["lon"].apply(pd.to_numeric, errors="coerce"), ) - .query("Status in @status_list") + .pipe(map_status, config=config) .pipe(lambda x: x[df.columns.intersection(config.get("target_columns"))]) .assign(Fueltype="Geothermal") .assign(Technology="Steam Turbine") @@ -2046,8 +2041,6 @@ def GWPT(raw=False, update=False, config=None): "Offshore mount unknown": "Offshore", } - status_list = config["GWPT"].get("status", ["operating"]) # noqa: F841 - df = df.rename(columns=RENAME_COLUMNS) df_final = ( df.pipe(set_column_name, "GWPT") @@ -2059,7 +2052,7 @@ def GWPT(raw=False, update=False, config=None): lat=df["lat"].apply(pd.to_numeric, errors="coerce"), lon=df["lon"].apply(pd.to_numeric, errors="coerce"), ) - .query("Status in @status_list") + .pipe(map_status, config=config) .pipe(lambda x: x[df.columns.intersection(config.get("target_columns"))]) .pipe(lambda x: x.replace({"Technology": technology_dict})) .assign(Fueltype="Wind") @@ -2111,8 +2104,6 @@ def GSPT(raw=False, update=False, config=None): "Assumed PV": "PV", } - status_list = config["GSPT"].get("status", ["operating"]) # noqa: F841 - df = df.rename(columns=RENAME_COLUMNS) df_final = ( df.pipe(set_column_name, "GSPT") @@ -2124,7 +2115,7 @@ def GSPT(raw=False, update=False, config=None): lat=df["lat"].apply(pd.to_numeric, errors="coerce"), lon=df["lon"].apply(pd.to_numeric, errors="coerce"), ) - .query("Status in @status_list") + .pipe(map_status, config=config) .pipe(lambda x: x[df.columns.intersection(config.get("target_columns"))]) .pipe(lambda x: x.replace({"Technology": technology_dict})) .assign(Fueltype="Solar") @@ -2201,16 +2192,8 @@ def classify_fuel(s): "no": "PP", } - status_list = config["GGPT"].get("status", ["operating"]) # noqa: F841 - df = df.rename(columns=RENAME_COLUMNS) - # conservative assumption that mothballed plants (without fixed retirement - # date) went out of operation in 2024 - mothballed_retirement = df["Status"].apply( - lambda x: 2024 if x == "mothballed" else np.nan - ) - df_final = ( df.pipe(set_column_name, "GGPT") .pipe(convert_to_short_name) @@ -2219,14 +2202,13 @@ def classify_fuel(s): DateIn=df["DateIn"].apply(pd.to_numeric, errors="coerce"), DateOut=df["DateOut"] .apply(pd.to_numeric, errors="coerce") - .combine_first(df["Planned retire"]) - .combine_first(mothballed_retirement), + .combine_first(df["Planned retire"]), lat=df["lat"].apply(pd.to_numeric, errors="coerce"), lon=df["lon"].apply(pd.to_numeric, errors="coerce"), Capacity=df["Capacity"].apply(pd.to_numeric, errors="coerce"), Fueltype=df.apply(classify_fuel, axis=1), ) - .query("Status in @status_list") + .pipe(map_status, config=config) .pipe(lambda x: x[df.columns.intersection(config.get("target_columns"))]) .pipe(lambda x: x.replace({"Technology": technology_dict})) .pipe(lambda x: x.replace({"Set": set_dict})) @@ -2279,7 +2261,6 @@ def GHPT(raw=False, update=False, config=None): "conventional and run-of-river": "Reservoir", "unknown": "Run-Of-River", } - status_list = config["GHPT"].get("status", ["operating"]) # noqa: F841 df = df.rename(columns=RENAME_COLUMNS) df_final = ( df.pipe(set_column_name, "GHPT") @@ -2291,7 +2272,7 @@ def GHPT(raw=False, update=False, config=None): lat=df["lat"].apply(pd.to_numeric, errors="coerce"), lon=df["lon"].apply(pd.to_numeric, errors="coerce"), ) - .query("Status in @status_list") + .pipe(map_status, config=config) .pipe(lambda x: x[df.columns.intersection(config.get("target_columns"))]) .pipe(lambda x: x.replace({"Technology": technology_dict})) .assign(Fueltype="Hydro") @@ -2436,9 +2417,7 @@ def _assign_PP_or_CHP(df): PARSE_COLUMNS + [ "Batterietechnologie", - "DatumBeginnVoruebergehendeStilllegung", "DatumEndgueltigeStilllegung", - "DatumWiederaufnahmeBetrieb", "EinheitBetriebsstatus", "EinheitMastrNummer", "Gemeinde", @@ -2477,8 +2456,6 @@ def _assign_PP_or_CHP(df): "ThermischeNutzleistung", "KwkMastrNummer", "Batterietechnologie", - "DatumBeginnVoruebergehendeStilllegung", - "DatumWiederaufnahmeBetrieb", "Postleitzahl", "Ort", "Gemeinde", @@ -2526,8 +2503,6 @@ def _assign_PP_or_CHP(df): if raw: return df - status_list = config["MASTR"].get("status", ["In Betrieb"]) # noqa: F841 - PLZ_map = PLZ_to_LatLon_map() df.Postleitzahl = ( df.Postleitzahl.astype(str) @@ -2555,7 +2530,7 @@ def _assign_PP_or_CHP(df): df_processed = ( df.rename(columns=RENAME_COLUMNS) - .query("Status in @status_list") + .pipe(map_status, config=config) .assign( projectID=lambda df: "MASTR-" + df.projectID, Name=lambda df: df.Name.combine_first(df.NameWindpark).combine_first( @@ -2566,15 +2541,7 @@ def _assign_PP_or_CHP(df): DateIn=lambda df: pd.to_datetime(df.DateIn).dt.year.combine_first( pd.to_datetime(df["GeplantesInbetriebnahmedatum"]).dt.year ), - DateOut=lambda df: pd.to_datetime(df.DateOut).dt.year.where( - df.Status != "Vorübergehend stillgelegt", - pd.to_datetime( - df["DatumBeginnVoruebergehendeStilllegung"] - ).dt.year.where( - df["DatumWiederaufnahmeBetrieb"].isna(), - pd.to_datetime(df.DateOut).dt.year, - ), - ), + DateOut=lambda df: pd.to_datetime(df.DateOut).dt.year, lat=lambda df: df.lat.combine_first(df.PLZ_lat), lon=lambda df: df.lon.combine_first(df.PLZ_lon), Duration=lambda df: df.StorageCapacity_MWh.div( @@ -2732,8 +2699,6 @@ def EESI( if raw: return df - status_list = config["EESI"].get("status_name", ["Operational"]) # noqa: F841 - RENAME_COLUMNS = { "title": "Name", "power": "Capacity", @@ -2748,7 +2713,7 @@ def EESI( df_processed = ( df.rename(columns=RENAME_COLUMNS) - .query("Status in @status_list") + .pipe(map_status, config=config) .assign( projectID=lambda df: "EESI-" + df.projectID.astype(str), StorageCapacity_MWh=lambda df: df.StorageCapacity_MWh.where( @@ -2839,8 +2804,6 @@ def GND( if raw: return df - status_list = config["GND"].get("status", ["Operational"]) # noqa: F841 - RENAME_COLUMNS = { "Id": "projectID", "Latitude": "lat", @@ -2851,7 +2814,7 @@ def GND( df_final = ( df.rename(columns=RENAME_COLUMNS) - .query("Status in @status_list") + .pipe(map_status, config=config) .assign( projectID=lambda df: "GND-" + df.projectID.astype(str), Capacity=lambda df: df.Capacity.where(df.Capacity > 0), @@ -2905,17 +2868,18 @@ def GHR( "ID": "projectID", "name": "Name", "country": "Country", - "Latitude": "plant_lat", - "Longitude": "plant_lon", + "plant_lat": "lat", + "plant_lon": "lon", "plant_type": "Technology", "dam_height_m": "DamHeight_m", "year": "DateIn", + "capacity_mw": "Capacity", } TECHNOLOGY_MAP = { "STO": "Reservoir", - "RTO": "Run-Of-River", - "PHS": "Pumped Hydro", - "canal": np.nan, + "ROR": "Run-Of-River", + "PS": "Pumped Storage", + "Canal": np.nan, } df_final = ( @@ -2923,7 +2887,6 @@ def GHR( .assign( projectID=lambda df: "GHR-" + df.projectID.astype(str), Name=lambda df: df.Name.str.split(" - ").str[0].combine_first(df.dam_name), - DateIn=lambda df: pd.to_datetime(df.DateIn).dt.year, Technology=lambda df: df.Technology.map(TECHNOLOGY_MAP), Volume_Mm3=lambda df: df.res_vol_km3 * 1e3, # StorageCapacity_MWh=lambda df: 9.81 * df.dam_height_m * df.Volume_Mm3 * 0.9 / 3.6, @@ -2932,6 +2895,7 @@ def GHR( Fueltype="Hydro", ) .pipe(clean_name) + .assign(Status="operating") .pipe(set_column_name, "GHR") .pipe(config_filter, config) ) diff --git a/powerplantmatching/matching.py b/powerplantmatching/matching.py index e1933d9f..0aadca65 100644 --- a/powerplantmatching/matching.py +++ b/powerplantmatching/matching.py @@ -262,6 +262,7 @@ def reduce_matched_dataframe(df, show_orig_names=False, config=None): """ Reduce a matched dataframe to a unique set of columns. For each entry take the value of the most reliable data source included in that match. + For the status, take the latest life-cycle stage given by any data source. Parameters ---------- @@ -280,9 +281,11 @@ def reduce_matched_dataframe(df, show_orig_names=False, config=None): {s: config[s]["reliability_score"] for s in sources}, dtype=float ).sort_values(ascending=False) cols = config["target_columns"] + stages = list(config["target_status"]) props_for_groups = {col: "first" for col in cols} props_for_groups.update( { + "Status": lambda x: max(x.dropna(), key=stages.index, default=np.nan), "DateIn": "min", "DateRetrofit": "max", "DateOut": "max", diff --git a/powerplantmatching/package_data/config.yaml b/powerplantmatching/package_data/config.yaml index 290fc401..f9626c03 100644 --- a/powerplantmatching/package_data/config.yaml +++ b/powerplantmatching/package_data/config.yaml @@ -12,12 +12,16 @@ google_api_key: # Note: querying "key == key" filters out nan values main_query: "Name != '' and (lat >= 30 or lat != lat)" +# Life-cycle stages to keep (see `target_status`), set `status` per data source to override +status: [announced, construction, operating, mothballed, retired] + # matching_sources, only the matched powerplants are added to the final dataset matching_sources: # Make individual queries for the datasets as done in `fully_included_sources` # Queries are combined with `main_query` with an `and` operator # capacity filters avoid matching of too small units (which is too time-consuming) # wind is per turbine rather than park in MASTR and unsuitable for matching + # projects (announced, construction) are not matched, as they often have the name and site of existing plants - ENTSOE: not (Country == 'Germany' and Fueltype == 'Wind') # wind in germany is provided by MASTR, other filters are due to large deviations to other datasets - GEO: Capacity >= 1 and not (Country == 'Germany' and Fueltype == 'Wind') and Fueltype not in ['Oil', 'Nuclear'] and not (Country in ['Bulgaria', 'Slovakia'] and Fueltype == 'Hard Coal') @@ -26,11 +30,11 @@ matching_sources: - JRC: Capacity >= 1 # wind in germany is provided by MASTR, other filters are due to large deviations to other datasets - OPSD: not (Country == 'Germany' and Fueltype == 'Wind') and ((Capacity >= 1 and Fueltype != 'Solar') or Capacity >= 3) and not (Country == 'Spain' and Fueltype == 'Hard Coal') and not (Country == 'Italy' and Fueltype == 'Natural Gas') - - BEYONDCOAL + - BEYONDCOAL: Status not in ['announced', 'construction'] # wind in germany is provided by MASTR - - GEM: Capacity >= 3 and not (Country == 'Germany' and Fueltype == 'Wind') + - GEM: Capacity >= 3 and not (Country == 'Germany' and Fueltype == 'Wind') and Status not in ['announced', 'construction'] # do not match units below 1 MW (2 MW for biogas, 3 MW for solar), exclude wind in Germany from any matching - - MASTR: (Fueltype != 'Wind') and ((Fueltype == 'Solar' and Capacity >= 3) or (Fueltype == 'Biogas' and Capacity >= 2) or (Fueltype not in ['Solar', 'Biogas'] and Capacity >= 1)) + - MASTR: (Fueltype != 'Wind') and ((Fueltype == 'Solar' and Capacity >= 3) or (Fueltype == 'Biogas' and Capacity >= 2) or (Fueltype not in ['Solar', 'Biogas'] and Capacity >= 1)) and Status not in ['announced', 'construction'] - OSM: Capacity >= 1 - EESI - GHR @@ -46,7 +50,6 @@ fully_included_sources: - MASTR: Fueltype != 'Nuclear' # take small hydro outside Germany from OPSD (highest coverage) - OPSD: Country != 'Germany' and Capacity < 1 and Capacity >= 0.1 and Fueltype == 'Hydro' - - BEYONDCOAL # include this selection of countries as they have poorer coverage in all other datasets - JRC: Country in ['Italy', 'Croatia', 'Serbia', 'Slovakia'] @@ -72,7 +75,6 @@ BEYONDCOAL: net_capacity: false aggregated_units: false reliability_score: 4 - status: ["construction", "operational", "no longer coal", "retired", "standby", "deactivated", "retrofitting"] fn: 20260731-BeyondFossilFuels-Europe_Coal_Plants_Database.xlsx url: https://tubcloud.tu-berlin.de/s/qHfNEELYy6EnBSX/download/20260731-BeyondFossilFuels-Europe_Coal_Plants_Database.xlsx IRENA: @@ -172,7 +174,6 @@ WEPP: GGPT: net_capacity: false reliability_score: 6 - status: ["operating", "retired", "construction"] fn: Global-Oil-and-Gas-Plant-Tracker-GOGPT-January-2026.xlsx url: https://tubcloud.tu-berlin.de/s/oTNEPzKcTZjdEZW/download/Global-Oil-and-Gas-Plant-Tracker-GOGPT-January-2026.xlsx GEM: @@ -182,64 +183,55 @@ GEM: GCPT: net_capacity: false reliability_score: 6 - status: ["operating", "retired", "construction", "mothballed"] fn: Global-Coal-Plant-Tracker-July-2026.xlsx url: https://tubcloud.tu-berlin.de/s/xyZSXcd4QNSzDsy/download/Global-Coal-Plant-Tracker-July-2026.xlsx GGTPT: net_capacity: false reliability_score: 6 aggregated_units: false - status: ["operating", "retired", "construction", "mothballed"] fn: Geothermal-Power-Tracker-March-2026-Final.xlsx url: https://tubcloud.tu-berlin.de/s/spGtyE3oPZrMeB3/download/Geothermal-Power-Tracker-March-2026-Final.xlsx GWPT: net_capacity: false reliability_score: 6 - status: ["operating", "retired", "construction"] fn: Global-Wind-Power-Tracker-February-2026.xlsx url: https://tubcloud.tu-berlin.de/s/mN7sbGDAQqd8yLD/download/Global-Wind-Power-Tracker-February-2026.xlsx GSPT: net_capacity: false reliability_score: 6 - status: ["operating", "retired", "construction"] fn: Global-Solar-Power-Tracker-February-2026.xlsx url: https://tubcloud.tu-berlin.de/s/Lx8KRcKBMPd2QrZ/download/Global-Solar-Power-Tracker-February-2026.xlsx GBPT: net_capacity: false reliability_score: 6 - status: ["operating", "retired", "construction"] fn: Global-Bioenergy-Power-Tracker-GBPT-V3.xlsx url: https://tubcloud.tu-berlin.de/s/ZkaQonLYdakrN75/download/Global-Bioenergy-Power-Tracker-GBPT-V3.xlsx GNPT: net_capacity: false reliability_score: 6 - status: ["operating", "retired", "mothballed", "construction"] fn: Global-Nuclear-Power-Tracker-September-2025.xlsx url: https://tubcloud.tu-berlin.de/s/NBX6G2eyZ3T97gq/download/Global-Nuclear-Power-Tracker-September-2025.xlsx GHPT: net_capacity: false reliability_score: 6 - status: ["operating", "retired", "construction"] fn: Global-Hydropower-Tracker-March-2026.xlsx url: https://tubcloud.tu-berlin.de/s/feKWmAfzPtHPHN2/download/Global-Hydropower-Tracker-March-2026.xlsx MASTR: net_capacity: true reliability_score: 7 - status: ["In Betrieb", "In Planung", "Endgültig stillgelegt", "Vorübergehend stillgelegt"] capacity_threshold: 0.05 # all values below will be filtered out, given in MW fn: dataversion-2026-08-16.zip url: https://tubcloud.tu-berlin.de/s/KJWoc8rwezoRkcQ/download/dataversion-2026-08-16.zip EESI: net_capacity: true reliability_score: 5 - status: ["Operational"] # since no start years given + status: [operating] # since no start years given fn: european-energy-storage-inventory-20260816-1026.json # from https://ses.jrc.ec.europa.eu/storage-inventory-tool/api/projects url: https://tubcloud.tu-berlin.de/s/ijiN2EWjHfg4NXf/download/european-energy-storage-inventory-20260816-1026.json GND: net_capacity: true reliability_score: 5 - status: ["Shutdown", "Operational", "Under Construction", "Decommissioning Completed"] url: https://raw.githubusercontent.com/cristianst85/GeoNuclearData/1bc8b4ac106af236902385b87e46c540b4864815/data/csv/denormalized/nuclear_power_plants.csv fn: nuclear_power_plants.csv GHR: @@ -265,6 +257,7 @@ target_columns: - Country - Capacity - Efficiency + - Status - DateIn - DateRetrofit - DateOut @@ -587,6 +580,46 @@ target_technologies: - på havet - op zee - zeewind +target_status: + # Map the status labels of the data sources to life-cycle stages (exact match, case-insensitive). + # Keep the stages in life-cycle order: if matched sources disagree, the later stage is used. + # Entries with labels which are not given (e.g. cancelled or shelved) are removed. + announced: + - announced + - pre-permit + - permitted + - pre-construction + - planned + - in planung + construction: + - construction + - under construction + - in bau + operating: + - operating + - operational + - in betrieb + - retrofitting + - special_case + mothballed: + - mothballed + - standby + - deactivated + - suspended operation + - inactive + - reserve + - shutdown_temporary + - seasonal_conservation + - vorübergehend stillgelegt + - betriebsstillstand + retired: + - retired + - no longer coal + - shutdown + - decommissioning completed + - endgültig stillgelegt + - stillgelegt + - rückgebaut clean_name: fueltypes_with_blocks: - Nuclear diff --git a/powerplantmatching/utils.py b/powerplantmatching/utils.py index 3ebb8b94..5c25aedf 100644 --- a/powerplantmatching/utils.py +++ b/powerplantmatching/utils.py @@ -120,6 +120,11 @@ def config_filter(df, config): cols = config["target_columns"] target_query = "Country in @countries and Fueltype in @fueltypes" + if "Status" in cols: + status = config.get(name, {}).get("status", config["status"]) + if unknown := set(status) - set(config["target_status"]): + raise ValueError(f"Unknown status {unknown} for {name}.") + target_query += " and (Status in @status or Status != Status)" main_query = config.get("main_query", "") diff --git a/test/test_cleaning.py b/test/test_cleaning.py index b4411075..ce980e4f 100644 --- a/test/test_cleaning.py +++ b/test/test_cleaning.py @@ -7,10 +7,14 @@ import pytest from powerplantmatching.cleaning import ( + aggregate_units, clean_name, gather_and_replace, gather_specifications, + map_status, ) +from powerplantmatching.core import get_config +from powerplantmatching.utils import config_filter, set_column_name TEST_DATA = { "Name": [ @@ -87,3 +91,77 @@ def test_clean_name(data): assert res.Name[2] == "Another Powerplant With Whitespaces" assert res.Name[3] == "Coalition" assert res.Name[4] == "Besonders Chp" + + +def test_map_status(): + status = ["Operating", "pre-permit", "Vorübergehend stillgelegt", "cancelled", None] + res = map_status(pd.DataFrame({"Status": status}), config=get_config()) + assert res.Status.to_list() == ["operating", "announced", "mothballed"] + + +@pytest.fixture +def status_data(): + return pd.DataFrame( + { + "Name": "Kraftwerk Musterstadt", + "Fueltype": "Hard Coal", + "Country": "Germany", + "Capacity": 400.0, + "Status": ["operating", "retired", np.nan], + "lat": 51.0, + "lon": 10.0, + "projectID": ["A", "B", "C"], + } + ).pipe(set_column_name, "GCPT") + + +@pytest.mark.parametrize( + "overall, source, expected", + [ + (None, None, ["operating", "retired", np.nan]), + (["operating"], None, ["operating", np.nan]), + (["operating"], ["retired"], ["retired", np.nan]), + ], +) +def test_config_filter_status(status_data, overall, source, expected): + config = get_config() + if overall is not None: + config["status"] = overall + if source is not None: + config["GCPT"]["status"] = source + res = config_filter(status_data, config) + pd.testing.assert_series_equal( + res.Status, pd.Series(expected, name="Status"), check_dtype=False + ) + + +def test_config_filter_unknown_status(status_data): + config = get_config() + config["status"] = ["Operational"] + with pytest.raises(ValueError, match="Unknown status"): + config_filter(status_data, config) + + +@pytest.mark.parametrize( + "fueltype, status, n_plants", + [ + ("Hard Coal", ["operating", "operating"], 1), + ("Hard Coal", ["operating", "retired"], 2), + ("Nuclear", ["operating", "operating"], 2), + ], +) +def test_aggregate_units(fueltype, status, n_plants): + config = get_config() + df = pd.DataFrame( + { + "Name": "Kraftwerk Musterstadt", + "Fueltype": fueltype, + "Country": "Germany", + "Capacity": 400.0, + "Status": status, + "lat": 51.0, + "lon": 10.0, + } + ).reindex(columns=config["target_columns"]) + res = aggregate_units(df, dataset_name="GCPT", config=config) + assert len(res) == n_plants diff --git a/test/test_matching.py b/test/test_matching.py new file mode 100644 index 00000000..31f8d356 --- /dev/null +++ b/test/test_matching.py @@ -0,0 +1,32 @@ +# SPDX-FileCopyrightText: Contributors to powerplantmatching +# +# SPDX-License-Identifier: MIT + +import numpy as np +import pandas as pd +import pytest + +from powerplantmatching.core import get_config +from powerplantmatching.matching import reduce_matched_dataframe + + +@pytest.mark.parametrize( + "status, expected", + [ + (["operating", "retired"], "retired"), + (["retired", "operating"], "retired"), + (["announced", "operating"], "operating"), + (["mothballed", np.nan], "mothballed"), + ([np.nan, np.nan], np.nan), + ], +) +def test_reduce_status_by_life_cycle(status, expected): + config = get_config() + sources = ["GCPT", "OPSD"] + columns = pd.MultiIndex.from_product([config["target_columns"], sources]) + df = pd.DataFrame(np.nan, index=[0], columns=columns, dtype=object) + df["Name"] = "Kraftwerk Musterstadt" + df.loc[0, "Status"] = status + res = reduce_matched_dataframe(df, config=config) + expected = pd.Series([expected], name="Status") + pd.testing.assert_series_equal(res.Status, expected, check_dtype=False)