EC-FR, EC-NL-CROP: override the EuroCrops rows that disagree with the taxonomy

CHANGELOG.md

@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ## [Unreleased]
 
 ### Added
+- Added `pixi run check-hcat`, which compares the HCAT mapping tables the converters read online with the taxonomy.
 - Added `FiboaDuckDBBaseConverter` for SQL-based conversion of large Parquet sources.
 - Added `PerFileBaseConverter` to process multi-file sources incrementally.
 - Added support for supplementary HCAT/crop mappings via `hcat_mapping_supplements`.
@@ -83,6 +84,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ### Fixed
 - Added HCAT spelling fixes via `csv_supplements` for DE-BB, DE-NDS and EC-SI.
+- EC-FR, EC-NL-CROP: HCAT mappings for iris, Jerusalem artichoke and parsley now match the taxonomy.
+- EuroCrops converters whose source already carries HCAT (such as EC-FR and EC-SI) now apply their supplements too.
 - Declared the `beautifulsoup4` dependency used by ES-PV and ES-VC.
 - REST converters:
   - Downloaded data cached for one dataset, edition or service is no longer served for another. Previously cached downloads are fetched again once.

fiboa_cli/datasets/commons/hcat.py

@@ -23,6 +23,8 @@ class AddHCATMixin:
     # Tables that fill gaps in the main one, for a country whose main table does
     # not carry every code the source uses. Rows here win where both carry a code.
     hcat_mapping_supplements: list[str] = []
+    # The column the supplements are keyed on when the source already carries HCAT
+    hcat_supplement_key = "crop:code"
     # Match on the crop name where the table has no row for the code:
     # be_wal_all_years.csv leaves original_code empty in 208 of its 298 rows.
     hcat_mapping_name_fallback = False
@@ -99,7 +101,9 @@ def add_hcat(self, gdf):
 
         # Lookup column that will be renamed after the migration to hcat:code
         hcat_code_column = next(k for k, v in self.hcat_columns.items() if v == "hcat:code")
-        if hcat_code_column not in gdf.columns:
+        if hcat_code_column in gdf.columns:
+            gdf = self.correct_resolved_hcat(gdf)
+        else:
             # Add HCAT columns based on crop-columns
             # Map to HCAT categories by using the mapping from the csv file
 
@@ -165,6 +169,21 @@ def map_by_name(attribute):
             )
         return gdf
 
+    def correct_resolved_hcat(self, gdf):
+        """The source carries HCAT already (EuroCrops); the supplements still win for their codes."""
+        rows = [row for url in self.hcat_mapping_supplements for row in load_hcat_mapping(url)]
+        if not rows:
+            return gdf
+        key = self.get_code_column(gdf, self.hcat_supplement_key).str.strip()
+        for column, attribute in zip(
+            self.hcat_columns.keys(), ("translated_name", "HCAT3_name", "HCAT3_code")
+        ):
+            fix = {row["original_code"].strip(): row[attribute] for row in rows}
+            hit = key.isin(fix.keys())
+            if column in gdf.columns and hit.any():
+                gdf.loc[hit, column] = key[hit].map(fix)
+        return gdf
+
     def post_migrate(self, gdf) -> gpd.GeoDataFrame:
         gdf = super().post_migrate(gdf)
         return self.add_hcat(gdf)

fiboa_cli/datasets/ec_fr.py

@@ -4,6 +4,7 @@
 
 class Converter(EuroCropsConverterMixin, FiboaBaseConverter):
     hcat_mapping_csv = "fr_2018.csv"
+    hcat_mapping_supplements = ["https://fiboa.org/code/fr/fr_2018_supplement.csv"]
     ec_year = 2018
     sources = {
         "https://zenodo.org/records/14094196/files/FR_2018.zip": ["FR_2018/FR_2018_EC21.shp"]

fiboa_cli/datasets/ec_nl_crop.py

@@ -4,6 +4,7 @@
 
 class NLEuroCropConverter(EuroCropsConverterMixin, NLCropConverter):
     hcat_mapping_csv = "nl_2020.csv"
+    hcat_mapping_supplements = ["https://fiboa.org/code/nl/nl_2020_supplement.csv"]
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)

fiboa_cli/datasets/ec_si.py

@@ -6,6 +6,7 @@ class Converter(EuroCropsConverterMixin, FiboaBaseConverter):
     area_is_in_ha = False
     hcat_mapping_csv = "si_2021.csv"
     hcat_mapping_supplements = ["https://fiboa.org/code/si/si_2021_supplement.csv"]
+    hcat_supplement_key = "crop_type_class"
     ec_year = 2021
     sources = {
         "https://zenodo.org/records/10118572/files/SI_2021.zip?download=1": ["SI_2021_EC21.shp"]
@@ -48,5 +49,5 @@ class Converter(EuroCropsConverterMixin, FiboaBaseConverter):
     }
 
     def add_hcat(self, gdf):
-        # skip adding hcat
-        return gdf
+        # the source carries HCAT and has no crop:code; only apply the supplement
+        return self.correct_resolved_hcat(gdf)

pyproject.toml

@@ -90,6 +90,8 @@ test = "pytest --cov=fiboa_cli --cov-report=term-missing --cov-report=html --cov
 lint = "ruff check --fix"
 format = "ruff format"
 check = {depends-on = ["lint", "format", "test"]}
+# Compare the HCAT mapping tables the converters read online with the taxonomy
+check-hcat = "python scripts/check_hcat.py"
 build = "python -m build"
 
 # Pre-commit tasks

scripts/check_hcat.py

@@ -0,0 +1,81 @@
+"""Check the HCAT mapping tables the converters read online against https://fiboa.org/code/hcat3.csv.
+
+The offline counterpart is tests/test_hcat_codes.py, which only sees the fixtures.
+Run with `pixi run check-hcat`; exits non-zero when a row disagrees with the taxonomy.
+"""
+
+import csv
+import io
+import sys
+from collections import defaultdict
+
+import requests
+
+from fiboa_cli.converters import Converters
+from fiboa_cli.datasets.commons.hcat import AddHCATMixin, hcat_mapping_url
+
+TAXONOMY = "https://fiboa.org/code/hcat3.csv"
+_tables = {}
+
+
+def fetch(url):
+    if url not in _tables:
+        try:
+            response = requests.get(url, timeout=60)
+        except requests.RequestException:
+            response = None
+        if response is None or response.status_code != 200:
+            _tables[url] = None
+        else:
+            try:
+                text = response.content.decode("utf-8")
+            except UnicodeDecodeError:
+                text = response.content.decode("latin-1")
+            _tables[url] = list(csv.DictReader(io.StringIO(text)))
+    return _tables[url]
+
+
+def row_key(row):
+    return (row.get("original_code") or "").strip() or (row.get("original_name") or "").strip()
+
+
+def main():
+    if fetch(TAXONOMY) is None:
+        print(f"unreachable: {TAXONOMY}")
+        return 1
+    taxonomy = {r["HCAT3_name"].strip(): r["HCAT3_code"].strip() for r in fetch(TAXONOMY)}
+    problems = defaultdict(set)
+    unreachable = defaultdict(set)
+    converters = Converters()
+    for _id in sorted(converters.list_ids()):
+        converter = converters.load(_id)
+        if not isinstance(converter, AddHCATMixin) or not converter.hcat_mapping_csv:
+            continue
+        rows = {}
+        for url in [converter.hcat_mapping_csv, *converter.hcat_mapping_supplements]:
+            table = fetch(hcat_mapping_url(url))
+            if table is None:
+                unreachable[hcat_mapping_url(url)].add(_id)
+                continue
+            rows |= {row_key(r): r for r in table}
+        for row in rows.values():
+            name = (row.get("HCAT3_name") or "").strip()
+            code = (row.get("HCAT3_code") or "").strip()
+            if not name and not code:
+                continue
+            if name not in taxonomy:
+                problems[f"{name!r} ({code}) is not an HCAT class"].add(_id)
+            elif taxonomy[name] != code:
+                problems[f"{name} has {code}, the taxonomy says {taxonomy[name]}"].add(_id)
+
+    for url, ids in sorted(unreachable.items()):
+        print(f"unreachable: {url} ({', '.join(sorted(ids))})")
+    for problem, ids in sorted(problems.items()):
+        print(f"{problem}: {', '.join(sorted(ids))}")
+    if not problems and not unreachable:
+        print("All mapping tables agree with the taxonomy.")
+    return 1 if problems or unreachable else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/data-files/convert/be_vlg/vlg_supplement.csv

@@ -0,0 +1,35 @@
+original_code,original_name,translated_name,HCAT3_name,HCAT3_code
+47,Goudsbloem,Marigold,calendula_marigold,3301061210
+48,Deder/huttentut,Camelina,camelina,3301061500
+391,Mengteelt andere wintergranen en winterteelt vlinderbloemige,Intercropping other winter cereals and winter legume,unspecified_cereals,3301011500
+392,Mengteelt zomergranen en zomerteelt vlinderbloemige,Intercropping spring cereals and spring legume,unspecified_cereals,3301011500
+395,Mengteelt wintertarwe of triticale en winterteelt vlinderbloemige,Intercropping winter wheat or triticale and winter legume,winter_common_soft_wheat,3301010101
+451,Wintervlas (olieproductie),Winter flax (oil production),flax_linseed_oil,3301060702
+511,Voedererwten (winterteelt) - niet menselijke consumptie,Fodder peas (winter) - not for human consumption,peas,3301020600
+512,Voedererwten (zomerteelt) - niet menselijke consumptie,Fodder peas (spring) - not for human consumption,peas,3301020600
+513,Drooggeoogste erwten (voor menselijke consumptie),Dry harvested peas (for human consumption),peas,3301020600
+521,Winterveldbonen - niet menselijke consumptie,Winter field beans - not for human consumption,beans,3301020100
+522,Zomerveldbonen - niet menselijke consumptie,Spring field beans - not for human consumption,beans,3301020100
+523,Drooggeoogste Vicia bonen (voor menselijke consumptie),Dry harvested Vicia beans (for human consumption),beans,3301020100
+524,Drooggeoogste Phaseolus bonen (voor menselijke consumptie),Dry harvested Phaseolus beans (for human consumption),beans,3301020100
+634,Mengsel met overwegend Japanse haver en vlinderbloemigen,Mixture predominantly Japanese oats with legumes,oats,3301010500
+635,Mengsel met overwegend Japanse haver zonder vlinderbloemigen,Mixture predominantly Japanese oats without legumes,oats,3301010500
+655,Sareptamosterd,Brown mustard,mustard,3301210100
+749,Mengsel met 1 of meerdere vlinderbloemigen,Mixture with one or more legumes,legumes_dried_pulses_protein_crops,3301020000
+858,Andere meerjarige kruiden - industrie,Other perennial herbs - industry,unspecified_aromatic_medicinal_culinary_plants_spices_herbs,3301061298
+881,Engelwortel - vers,Angelica - fresh,angelica,3301061204
+899,Rotatieteelt van groentegewassen (vb pluktuin),Rotational cultivation of vegetable crops,fresh_vegetables,3301070000
+907,Aardappelen (gecertificeerd pootgoed),Potatoes (certified seed),potatoes,3301030000
+908,Aardappelen (hoevepootgoed),Potatoes (farm-saved seed),potatoes,3301030000
+923,Wintervlas (vezelproductie en industriële toepassingen),Winter flax (fibre production and industrial use),flax_linen,3301060701
+933,Kikkererwten,Chickpeas,chickpeas,3301020200
+934,Linzen,Lentils,lentils,3301020500
+6041,Zaaizaad wintervlas,Seed of winter flax,flax_linen,3301060701
+8526,Chinese kool - industrie,Chinese cabbage - industry,chinese_cabbage,3301210205
+8531,Radijs - industrie,Radish - industry,radish,3301290600
+8532,Rode biet - industrie,Beetroot - industry,beetroot_beets,3301290200
+8545,Raketsla - Rucola - industrie,Rocket - industry,rocket_arugula,3301310600
+8552,Tomaten - industrie,Tomatoes - industry,tomato,3301280000
+8553,Paprika - industrie,Bell pepper - industry,bell_pepper_paprika,3301300100
+8581,Andere meerjarige kruiden - vers,Other perennial herbs - fresh,aromatic_medicinal_culinary_plants_spices_herbs,3301061200
+9571,Begonia voor de knol,Begonia for the tuber,begonias,3301080400

tests/data-files/convert/cz/cz_supplement.csv

@@ -167,3 +167,5 @@ original_code,original_name,translated_name,HCAT3_name,HCAT3_code
 50371,Směs pro ozeleněný kolejový řádek,Mixture for a greened tramline,not_known_and_other,3399000000
 50372,Směs pro ochranný pás kolem KP,Protective strip mixture around a landscape feature,not_known_and_other,3399000000
 50374,Chřestnatec,Unidentified species (chřestnatec),not_known_and_other,3399000000
+389,Petržel kořenová,Petroselinum,parsly,3301061227
+390,Petržel naťová,Parsley,parsly,3301061227

tests/data-files/convert/de_bb/de_supplement.csv

@@ -0,0 +1,3 @@
+original_code,original_name,translated_name,HCAT3_name,HCAT3_code
+659,Petersilie,parsley/Petroselinum (parsley),parsly,3301061227
+860,Spargel,asparagus,asparagus,3301200000

tests/data-files/convert/dk/dk_supplement.csv

@@ -0,0 +1,32 @@
+original_code,original_name,translated_name,HCAT3_name,HCAT3_code
+19,Majs til modenhed med græsudlæg,Maize for maturity with grass undersowing,grain_maize_corn_popcorn,3301010600
+26,Linser,Lentils,lentils,3301020500
+41,Spindhør,Fibre flax,flax_linen,3301060701
+70,"Vårbyg, efterårssået","Spring barley, autumn sown",spring_barley,3301010402
+71,"Vårhvede, efterårssået","Spring wheat, autumn sown",spring_common_soft_wheat,3301010102
+72,"Vårhavre, efterårssået","Spring oats, autumn sown",spring_oats,3301010502
+127,Lucernefrø,Alfalfa seed,alfalfa_lucerne,3301090301
+154,"Kartofler, spise- (proces, skrællet kogte)","Potatoes, table (processed, peeled boiled)",potatoes,3301030000
+155,"Kartofler, pulver/granules-","Potatoes, powder/granules",potatoes,3301030000
+156,"Kartofler, friteret/chips/pommes frites","Potatoes, fried/chips",potatoes,3301030000
+157,"Kartofler, spise- tidligt høstede med efterafgrøder","Potatoes, table, early harvested with catch crops",potatoes,3301030000
+218,Majshelsæd med græsudlæg,Maize wholecrop with grass undersowing,green_silo_maize,3301090400
+224,"Blandkorn, efterårssået helsæd","Meslin, autumn sown wholecrop",winter_meslin,3301011101
+236,"Græs med kløver/lucerne, under 50 % bælgpl. (omdrift) efterårsudlagt i vinterkorn til grønkorn","Grass with clover/alfalfa, under 50% legume (rotation), autumn sown in winter cereal for green fodder",plants_harvested_green,3301090000
+237,"Græs med kløver/lucerne, over 50 % bælgpl. (omdrift) efterårsudlagt i vinterkorn til grønkorn","Grass with clover/alfalfa, over 50% legume (rotation), autumn sown in winter cereal for green fodder",clover,3301090303
+345,"Brak langs vandløb og søer, slåning (alternativ til efterafgrøder)","Fallow along watercourses and lakes, mown",fallow_land_not_crop,3301110000
+425,Sukkermajs med græsudlæg,Sweetcorn with grass undersowing,grain_maize_corn_popcorn,3301010600
+426,"Bønner, andre","Beans, other",beans,3301020100
+427,Babyleaves,Baby leaves,salads_lettuce_leaf_vegetables,3301310000
+428,Spidskål,Pointed cabbage,savoy_cabbage,3301210211
+482,Skovlandbrug med permanent græs,Agroforestry with permanent grass,tree_wood_forest,3306000000
+483,Skovlandbrug med græs i omdrift,Agroforestry with grass in rotation,tree_wood_forest,3306000000
+484,Skovlandbrug med omdriftsafgrøder,Agroforestry with rotational crops,tree_wood_forest,3306000000
+485,Skovlandbrug med permanente afgrøder,Agroforestry with permanent crops,tree_wood_forest,3306000000
+490,"Hassel, træ (Corylus avellana)","Hazel, tree (Corylus avellana)",hazelnuts_hazel,3303030200
+491,Storfrugtet tranebær,Large cranberry,cranberry,3303020500
+496,"Medicinpl., vedplanter","Medicinal plants, woody",aromatic_medicinal_culinary_plants_spices_herbs,3301061200
+535,Bærmispel,Juneberry,amelanchier_serviceberry,3303010100
+565,"Skovrejsning, direktivimplementerende uden tilsagn","Afforestation, directive-implementing without commitment",afforestation_reforestation,3306010000
+566,"Klimaskovrejsning, national ordning","Climate afforestation, national scheme",afforestation_reforestation,3306010000
+575,Skovrejsning (privat) - kulstofbinding og grundvandsbeskyttelse,Afforestation (private) - carbon capture and groundwater protection,afforestation_reforestation,3306010000

tests/data-files/convert/ec_fr/fr_2018_supplement.csv

@@ -0,0 +1,2 @@
+original_code,original_name,translated_name,HCAT3_name,HCAT3_code
+TOP,Topinambour,Jerusalem artichoke,topinambur_jerusalem_artichoke,3301180000