From 553d7ba6ac66f32b2f8891e9c709b006361ac2da Mon Sep 17 00:00:00 2001 From: Ivor Bosloper Date: Thu, 24 Sep 2026 21:50:29 +0200 Subject: [PATCH 1/3] EC-BE-VLG converts again, and ES maps its crops to HCAT - ec_be_vlg: drop the variants inherited from be_vlg, which vecorel refuses next to sources - es: map PARC_PRODUCTO to HCAT with fiboa.org/code/es/es.csv; the old code list URL was a 404 - test that no converter declares both sources and variants Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 ++ fiboa_cli/datasets/commons/hcat.py | 2 +- fiboa_cli/datasets/ec_be_vlg.py | 2 ++ fiboa_cli/datasets/es.py | 13 +++++++------ tests/test_converters.py | 6 ++++++ 5 files changed, 18 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ada43c7..f0b5d0f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - US-CSB: Editions now cover 2017-2024. ### Fixed +- EC-BE-VLG: converting no longer fails on the variants inherited from BE-VLG. +- ES: crops are mapped to HCAT, and `crop:code_list` points to a code list that exists. - Added HCAT spelling fixes via `csv_supplements` for DE-BB, DE-NDS and EC-SI. - Declared the `beautifulsoup4` dependency used by ES-PV and ES-VC. - REST converters: diff --git a/fiboa_cli/datasets/commons/hcat.py b/fiboa_cli/datasets/commons/hcat.py index 926ee7b6..cfe9328f 100644 --- a/fiboa_cli/datasets/commons/hcat.py +++ b/fiboa_cli/datasets/commons/hcat.py @@ -126,7 +126,7 @@ def map_by_name(attribute): if name_col is not None: col = col.fillna(name_col.str.strip().map(map_by_name(v))) gdf[k] = col - assert np.unique(col[~col.isna()]).size > 1, "No HCAT crops mapped" + assert np.unique(col[~col.isna()]).size > 0, "No HCAT crops mapped" if col is not None and col.isna().any(): index = [ diff --git a/fiboa_cli/datasets/ec_be_vlg.py b/fiboa_cli/datasets/ec_be_vlg.py index 5f1c048c..9eb0b6c8 100644 --- a/fiboa_cli/datasets/ec_be_vlg.py +++ b/fiboa_cli/datasets/ec_be_vlg.py @@ -9,6 +9,8 @@ class ECConverter(EuroCropsConverterMixin, BEVLGBaseConverter): "BE_VLG_2021/BE_VLG_2021_EC21.shp" ] } + # the single EuroCrops 2021 release, not the parent's yearly editions + variants = {} def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/fiboa_cli/datasets/es.py b/fiboa_cli/datasets/es.py index 64cc3e3d..680fdff8 100644 --- a/fiboa_cli/datasets/es.py +++ b/fiboa_cli/datasets/es.py @@ -4,9 +4,10 @@ from vecorel_cli.vecorel.extensions import ADMIN_DIVISION from ..conversion.fiboa_converter import FiboaBaseConverter +from .commons.hcat import AddHCATMixin -class Converter(FiboaBaseConverter): +class Converter(AddHCATMixin, FiboaBaseConverter): id = "es" short_name = "Spain" title = "Spain Declared Crops (Cultivos Declarados SIGPAC)" @@ -25,6 +26,9 @@ class Converter(FiboaBaseConverter): variants = {"2025": "2025"} + # FEGA declared-crop codes (PARC_PRODUCTO) to HCAT; the mixin also publishes it as crop:code_list + hcat_mapping_csv = "https://fiboa.org/code/es/es.csv" + columns = { "geometry": "geometry", "id": "id", @@ -45,14 +49,11 @@ class Converter(FiboaBaseConverter): column_additions = { "admin:country_code": "ES", - # FEGA declared-crop codelist (PARC_PRODUCTO) — separate from the SIGPAC land-use list. - # Reference list shipped inside each provincial GPKG as the `cod_producto` layer. - "crop:code_list": "https://fiboa.org/code/es/cultivos_declarados/parc_producto.csv", } column_migrations = { - # crop:code must be a string per the crop extension; parc_producto is an integer. - "parc_producto": lambda col: col.astype("Int64").astype(str), + # crop:code must be a string per the crop extension; 0 is "unknown product" in the code list + "parc_producto": lambda col: col.astype("Int64").fillna(0).astype(str), # admin_*_code are strings; zero-pad province to 2 digits (INE convention). "provincia": lambda col: col.astype("Int64").astype(str).str.zfill(2), "municipio": lambda col: col.astype("Int64").astype(str), diff --git a/tests/test_converters.py b/tests/test_converters.py index d84c9f84..0e47d865 100644 --- a/tests/test_converters.py +++ b/tests/test_converters.py @@ -453,3 +453,9 @@ def square(x, side): # one value for every row, so it is written to the collection metadata collection = json.loads(pq.ParquetFile(tmp_parquet_file).schema_arrow.metadata[b"collection"]) assert collection["determination:datetime"] == "2025-01-01T00:00:00Z" + + +def test_no_converter_declares_both_sources_and_variants(): + c = Converters() + for _id in c.list_ids(): + c.load(_id)._require_one_source_of_urls() From f9c916c88a6adacfd9e57a61219d9a3785c9b620 Mon Sep 17 00:00:00 2001 From: Ivor Bosloper Date: Fri, 25 Sep 2026 12:34:41 +0200 Subject: [PATCH 2/3] IE-LPIS: read the HCAT mapping under its new attribute name Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + fiboa_cli/datasets/ie_lpis.py | 2 +- tests/test_converters.py | 10 ++++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0b5d0f1..f8e124ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,6 +75,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed - EC-BE-VLG: converting no longer fails on the variants inherited from BE-VLG. - ES: crops are mapped to HCAT, and `crop:code_list` points to a code list that exists. +- IE-LPIS: the HCAT mapping is read again; it still used the attribute name from before the rename. - Added HCAT spelling fixes via `csv_supplements` for DE-BB, DE-NDS and EC-SI. - Declared the `beautifulsoup4` dependency used by ES-PV and ES-VC. - REST converters: diff --git a/fiboa_cli/datasets/ie_lpis.py b/fiboa_cli/datasets/ie_lpis.py index e37e8a7a..189e5671 100644 --- a/fiboa_cli/datasets/ie_lpis.py +++ b/fiboa_cli/datasets/ie_lpis.py @@ -77,7 +77,7 @@ class Converter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): provider = "Department of Agriculture, Food and the Marine " attribution = "Ireland Department of Agriculture, Food and the Marine" license = "CC-BY-4.0" - ec_mapping_csv = ( + hcat_mapping_csv = ( "https://fiboa.org/code/ie/ie.csv" # the GSAA list, extended by the LPIS-only names ) area_is_in_ha = False diff --git a/tests/test_converters.py b/tests/test_converters.py index 0e47d865..22c97b17 100644 --- a/tests/test_converters.py +++ b/tests/test_converters.py @@ -459,3 +459,13 @@ def test_no_converter_declares_both_sources_and_variants(): c = Converters() for _id in c.list_ids(): c.load(_id)._require_one_source_of_urls() + + +def test_no_converter_uses_the_old_hcat_attribute_names(): + # renamed in #320; a converter still setting ec_mapping* is silently ignored + c = Converters() + old = { + _id: [a for a in dir(type(c.load(_id))) if a.startswith("ec_mapping")] + for _id in c.list_ids() + } + assert not {k: v for k, v in old.items() if v} From 944f8a05d419a0c7266385e5995747ced7793a7d Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Fri, 25 Sep 2026 13:18:52 +0200 Subject: [PATCH 3/3] Fix changelog --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8e124ef..271055fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,9 +73,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - US-CSB: Editions now cover 2017-2024. ### Fixed -- EC-BE-VLG: converting no longer fails on the variants inherited from BE-VLG. -- ES: crops are mapped to HCAT, and `crop:code_list` points to a code list that exists. -- IE-LPIS: the HCAT mapping is read again; it still used the attribute name from before the rename. - Added HCAT spelling fixes via `csv_supplements` for DE-BB, DE-NDS and EC-SI. - Declared the `beautifulsoup4` dependency used by ES-PV and ES-VC. - REST converters: @@ -101,6 +98,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - DK: - IDs are now derived from `Journalnr` and `Marknr` together (or the row number in editions without either), because `Marknr` alone repeats across holdings. - Missing crop codes are kept empty instead of being filled with the undefined code 0. +- EC-BE-VLG: Converting no longer fails on the variants inherited from BE-VLG. - EC-EE: Fixed shapefile naming and year-column migration. - EC-FR: Added the missing 2018 RPG campaign from EuroCrops. - EC-LT: @@ -110,6 +108,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - EC-SI: Relaxed requirements to match fields present in source data. - EE: Published valid crop code, land-use class and stable parcel identifier; cache files are now campaign-specific. - ES: + - ES now maps crops to HCAT, and `crop:code_list` points to a code list that exists. - ES-AN now uses the correct land-use column and campaign-based determination date. - ES-CAT and ES-CN now map crop codes to HCAT with the extended mapping table. - ES-CB now derives determination date from the campaign. @@ -125,6 +124,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Crop codes introduced after the 2018 EuroCrops table (for example JAC, the most common code of 2024) now map to HCAT; 7.50% of the 2024 fields were unmapped before. - `id` is the RPG parcel id plus a part number where it repeats (multipart splits, reissued ids); the source value is kept as `parcel_id`. - IE: Uses stable feature IDs and publishes computed `metrics:area` when missing from source. +- IE-LPIS: The HCAT mapping is read again; it still used the attribute name from before the rename. - JP: Uses campaign-specific determination dates and DuckDB conversion path. - LT: Updated to Europe-LAND v1.3 with 2025 coverage. - SK: Fixed edition selection, crop-name matching and block/id handling across campaigns.