diff --git a/CHANGELOG.md b/CHANGELOG.md index 1169f23d..0ada43c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,7 +37,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - `fiboa publish` no longer uploads to S3 or generates README/LICENSE files. It now creates GeoParquet, PMTiles and a STAC Collection with relative links, checksums and web-map-links. - Updated `aiohttp` to support Zenodo responses that include both returned `Content-Type` headers. - Improved geometry axis handling so generated tiles and bounding boxes keep x/y order consistent in output. -- Updated vecorel-cli to 0.3.0, including improved validation defaults, latest-variant selection when `--variant` is not provided, multi-volume 7z download support, and more. +- Updated vecorel-cli to 0.3.1, including improved validation defaults, latest-variant selection when `--variant` is not provided, multi-volume 7z download support, Deflate64 ZIP extraction, and more. - REST converters download much faster from large layers and retry when a service answers with intermittent errors. - Converters no longer split multi-part geometries into one row per polygon: fields keep the geometry modeling of the source (Polygon or MultiPolygon), the source-published area and perimeter, and one id per source feature. Use `fiboa improve --explode-geometries` when single polygons are needed. - Converters whose variants are years (1900-2100) now fill `determination:datetime` from the selected year unless they provide a determination date themselves. @@ -64,6 +64,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - URL discovery now requires all nine regional GeoPackages to avoid partial campaign publication. - NL: Extended BRP coverage to 2009-2026 and moved to the newer PDOK source. - PT: Updated for the 2025 edition and its schema/unit changes. +- PT: Editions now cover 2017-2025. + - 2020-2022 merge the regional files, which come in four projections; 2020 and 2021 join the crop code from a separate table. + - 2017-2019 publish the crop as a Portuguese name, which is resolved to a code through pt.csv and kept as `crop:name`. + - Perimeters are measured in each feature's UTM zone. - SE: Editions now cover 2015-2025 from the yearly WFS filter. - SI: Extended editions back to 2019. - US-CSB: Editions now cover 2017-2024. diff --git a/fiboa_cli/datasets/pt.py b/fiboa_cli/datasets/pt.py index 4dbb6931..65f03a50 100644 --- a/fiboa_cli/datasets/pt.py +++ b/fiboa_cli/datasets/pt.py @@ -1,15 +1,165 @@ +import os import re +import unicodedata import geopandas as gpd +import numpy as np +import pandas as pd +import pyogrio from vecorel_cli.conversion.admin import AdminConverterMixin from ..conversion.fiboa_converter import FiboaBaseConverter -from .commons.hcat import AddHCATMixin +from .commons.hcat import AddHCATMixin, load_hcat_mapping # Up to 2023 the country is split into "Culturas_" layers, from 2025 into # "T" layers. Both files carry other layers too (parcel blocks, land cover, # an empty "Culturas" container, a non-spatial "Codes" table) that are not field boundaries. DATA_LAYER = re.compile(r"^(Culturas_.+|T[0-9A-Z]{3})$") +# 2021 also has "Culturas_2021", a table without geometry, which DATA_LAYER would select +EDITION_LAYER = { + "2022": re.compile(r"^Ocupacoes_solo"), + "2021": re.compile(r"^Ocupacoes_solo_"), + "2020": re.compile(r"^Subparcelas"), +} + +# Accented names are globbed: they extract differently depending on the tool +MEMBERS = { + "2022": [ + "Continente.gpkg", + "Madeira.gpkg", + "Ca_22_Acores_Ocidental.gpkg", + "Ca_22_Acores_Oriental_Central.gpkg", + ], + "2021": [ + "Ocupacoes_solo_ALENTEJO.shp", + "Ocupacoes_solo_ALGARVE.shp", + "Ocupacoes_solo_AREA_METROPOLITANA_DE_LISBOA.shp", + "Ocupacoes_solo_Centro.shp", + "Ocupacoes_solo_Norte.shp", + "Ocupacoes_solo__acores_central_oriental.shp", + "Ocupacoes_solo__acores_ocidental.shp", + "Ocupacoes_solo_madeira.shp", + ], + "2020": [ + "Subparcelas_ALENTEJO.shp", + "SubparcelasALGARVE.shp", + "SubparcelasCENTRO.shp", + "SubparcelasNORTE.shp", + "Subparcelas_A*ores_Central_Oriental.shp", + "Subparcelas_A*ores_Ocidental.shp", + "Subparcelas_Madeira.shp", + "Subparcelas*REA_METROPOLITANA_DE_LISBOA.shp", + ], + "2019": [ + "Ocupacoes_solo_AML.shp", + "Ocupacoes_solo_Alentejo.shp", + "Ocupacoes_solo_Algarve.shp", + "Ocupacoes_solo_Centro_N.shp", + "Ocupacoes_solo_Centro_S.shp", + "Ocupacoes_solo_Norte_S.shp", + "Ocupacoes_solo_RAA.shp", + "Ocupacoes_solo_RAM.shp", + "ocupacoes_solo_n_1.shp", + "ocupacoes_solo_n_2.shp", + ], + "2018": [ + "Ocupacoes_solo_AML.shp", + "Ocupacoes_solo_Alentejo.shp", + "Ocupacoes_solo_Algarve.shp", + "Ocupacoes_solo_Centro_N.shp", + "Ocupacoes_solo_Centro_S.shp", + "Ocupacoes_solo_Norte_N.shp", + "Ocupacoes_solo_Norte_S.shp", + "Ocupacoes_solo_RAA.shp", + "Ocupacoes_solo_RAM.shp", + "ocupacoes.solo.Norte_N1.2018jun10.shp", + ], + "2017": [ + "Ocupacoes_solo_AML.shp", + "Ocupacoes_solo_Alentejo.shp", + "Ocupacoes_solo_Algarve.shp", + "Ocupacoes_solo_Centro_N.shp", + "Ocupacoes_solo_Centro_S.shp", + "Ocupacoes_solo_Norte_S.shp", + "Ocupacoes_solo_RAA.shp", + "Ocupacoes_solo_RAM.shp", + "ocupacoes_solo_norte_n1.shp", + "ocupacoes_solo_norte_n2.shp", + ], +} + +# 2020 and 2021 publish the crop code in a separate table +CROP_TABLE = {"2021": "Culturas_2021.dbf", "2020": "culturas_2020.dbf"} + +# 2017-2019 publish crop names instead of codes +NAME_EDITIONS = ("2019", "2018", "2017") + +# Count, case and numbering of the crop columns vary per file; the lowest is the primary crop +CROP_COLUMN = re.compile(r"^[Cc](\d{1,2})$") + +# 2018 publishes 154,980 fields in both files; Norte_S also has fields of its own +OVERLAPPING_MEMBERS = {"2018": ("Ocupacoes_solo_Norte_N", "Ocupacoes_solo_Norte_S")} + +# 2017's island files publish no field id; numbered from here, far above any OSA_ID (~46M) +ISLAND_ID_BASE = 10**12 + +# 2018 replaced accented letters in some crop names with "?", resolved offline against pt.csv +QUESTION_MARK_ALIASES = { + "AGRI?O": "AGRIAO", + "AVEL?": "AVELA", + "CONSOCIAC?ES ANUAIS E OUTRAS CULT FORRAG ANUAIS": ( + "CONSOCIACOES ANUAIS E OUTRAS CULT FORRAG ANUAIS" + ), + "EP BOSQUETE E FORMAC?ES RELIQUIAIS AREA UTIL": ( + "EP BOSQUETE E FORMACOES RELIQUIAIS AREA UTIL" + ), + "FEIJ?O": "FEIJAO", + "GR?O DE BICO": "GRAO DE BICO", + "LIM?O": "LIMAO", + "MAC?": "MACA", + "MACICOS OU FORMAC?ES RELIQUIAIS OU NOTAVEIS": ("MACICOS OU FORMACOES RELIQUIAIS OU NOTAVEIS"), + "MEL?O": "MELAO", + "PINH?O": "PINHAO", + "ROM?": "ROMA", + "SOBREIRO PARA PRODUC?O DE CORTICA": "SOBREIRO PARA PRODUCAO DE CORTICA", + "SUPERFICIE ARBUSTIVA N?O PASTOREAVEL": "SUPERFICIE ARBUSTIVA NAO PASTOREAVEL", +} + +# Dropping the other columns per file keeps millions of unused strings out of memory +KEEP = ("geometry", "OSA_ID", "PAR_ID", "PAR_NUM", "C1", "member") + + +def normalise_crop_name(value, keep_question_mark=False): + """Accents and separators vary within an edition (PRADOS_TEMPORARIOS, PRADOS TEMPORÁRIOS)""" + text = unicodedata.normalize("NFKD", str(value)) + text = "".join(c for c in text if not unicodedata.combining(c)).upper() + allowed = "A-Z0-9 ?" if keep_question_mark else "A-Z0-9 " + text = re.sub(f"[^{allowed}]+", " ", text) + return re.sub(r"\s+", " ", text).strip() + + +def read_crop_table(path): + # 2021 spells the key Osa_id and types it as a float + key = next(f for f in pyogrio.read_info(path)["fields"] if f.lower() == "osa_id") + df = pyogrio.read_dataframe(path, columns=[key, "C1"], read_geometry=False) + df = df.rename(columns={key: "OSA_ID"}) + # 2021 has 14 empty rows keyed 0, which no field uses + return df[df["OSA_ID"] != 0].astype({"OSA_ID": "int64"}) + + +def perimeter_metres(geometry): + """Measured in each feature's UTM zone: EPSG:6933 distorts lengths by up to ±10% and + Portugal spans zones 25N to 29N.""" + x = geometry.representative_point().x.to_numpy(dtype="float64") + # rows without a geometry stay NaN; the base converter drops them + located = np.isfinite(x) + zones = np.zeros(len(x), dtype="int64") + zones[located] = 32600 + (np.floor((x[located] + 180) / 6) + 1).astype("int64") + out = np.full(len(x), np.nan) + for zone in np.unique(zones[located]): + in_zone = zones == zone + out[in_zone] = geometry[in_zone].to_crs(f"EPSG:{zone}").length.to_numpy() + return out class PTConverter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): @@ -22,20 +172,17 @@ class PTConverter(AdminConverterMixin, AddHCATMixin, FiboaBaseConverter): variants = { "2025": BASE + "2025/culturas.gpkg", "2023": BASE + "2023/Continente.gpkg", - "2022": BASE + "2022/2022.zip", - "2021": BASE + "2021/2021.zip", - "2020": BASE + "2017-2020/2020.zip", - "2019": BASE + "2017-2020/2019.zip", - "2018": BASE + "2017-2020/2018.zip", - "2017": BASE + "2017-2020/2017.zip", + "2022": {BASE + "2022/2022.zip": MEMBERS["2022"]}, + "2021": {BASE + "2021/2021.zip": MEMBERS["2021"]}, + "2020": {BASE + "2017-2020/2020.zip": MEMBERS["2020"]}, + "2019": {BASE + "2017-2020/2019.zip": MEMBERS["2019"]}, + "2018": {BASE + "2017-2020/2018.zip": MEMBERS["2018"]}, + "2017": {BASE + "2017-2020/2017.zip": MEMBERS["2017"]}, "2016": BASE + "2011_2016/2016.zip", "2015": BASE + "2011_2016/2015.zip", # ... } - def layer_filter(self, layer, uri): - return bool(DATA_LAYER.match(layer)) - provider = ( "IPAP - Instituto de Financiamento da Agricultura e Pescas " ) @@ -48,7 +195,6 @@ def layer_filter(self, layer, uri): "CUL_ID": "id", "OSA_ID": "block_id", "CUL_CODIGO": "crop:code", - # The crop name is only published up to 2023; from 2025 the code is all there is. "CT_português": "crop:name", "Shape_Area": "metrics:area", "Shape_Length": "metrics:perimeter", @@ -62,22 +208,119 @@ def layer_filter(self, layer, uri): } } + def layer_filter(self, layer, uri): + # 2017-2019 are single-layer shapefiles selected by MEMBERS alone + if self.variant in NAME_EDITIONS: + return True + return bool(EDITION_LAYER.get(self.variant, DATA_LAYER).match(layer)) + + def file_migration(self, gdf, path, uri, layer=None): + if self.variant not in MEMBERS: + return gdf + # the regions come in four projections + gdf = gdf.to_crs("EPSG:4326") + gdf["member"] = layer or os.path.splitext(os.path.basename(path))[0] + if self.variant in NAME_EDITIONS: + numbered = {int(m[1]): c for c in gdf.columns if (m := CROP_COLUMN.match(str(c)))} + if numbered: + gdf = gdf.rename(columns={numbered[min(numbered)]: "C1"}) + return gdf[[c for c in KEEP if c in gdf.columns]] + + def read_data(self, paths, **kwargs): + gdf = super().read_data(paths, **kwargs) + if self.variant in CROP_TABLE: + crops = read_crop_table( + os.path.join(os.path.dirname(paths[0][0]), CROP_TABLE[self.variant]) + ) + gdf["OSA_ID"] = gdf["OSA_ID"].astype("int64") + gdf = gdf.merge(crops, on="OSA_ID", how="left", validate="many_to_one") + return gdf + def migrate(self, gdf) -> gpd.GeoDataFrame: # 2025 renamed the crop code column and dropped the crop name. if "PUN_CUL_CO" in gdf.columns: gdf = gdf.rename(columns={"PUN_CUL_CO": "CUL_CODIGO"}) + if self.variant in MEMBERS: + gdf = self._to_2023_layout(gdf) + + # Shape_* are in degrees: 2025 is published in WGS 84, 2017-2022 are reprojected to it. + # The base converter measures the area (area_calculate_missing). if gdf.crs is not None and gdf.crs.is_geographic: - # 2025 is published in WGS 84, with Shape_Area and Shape_Length computed in - # degrees. Recompute both in metres; up to 2023 the file is in ETRS89 / - # Portugal TM06 and the published values are already metric. - metric = gdf.geometry.to_crs("EPSG:6933") - gdf["Shape_Area"] = metric.area - gdf["Shape_Length"] = metric.length - - # 2025 types the identifiers as floats, which would stringify id as "28398800.0". + gdf = gdf.drop(columns=["Shape_Area"], errors="ignore") + gdf["Shape_Length"] = perimeter_metres(gdf.geometry) + + # float ids would stringify as "1.0" for column in ("OSA_ID", "CUL_ID"): if column in gdf.columns and gdf[column].dtype.kind == "f": gdf[column] = gdf[column].astype("int64") return super().migrate(gdf) + + def _to_2023_layout(self, gdf): + if self.variant in NAME_EDITIONS: + gdf = self._from_name_edition(gdf) + else: + # no crop parcels are published: the land occupation is the field, its parcel the block + gdf = gdf.rename(columns={"C1": "CUL_CODIGO", "OSA_ID": "CUL_ID", "PAR_ID": "OSA_ID"}) + # crop:code is required and cannot be written as null + gdf["CUL_CODIGO"] = gdf["CUL_CODIGO"].fillna("") + return gdf.drop(columns="member") + + def _from_name_edition(self, gdf): + kept, repeated = OVERLAPPING_MEMBERS.get(self.variant, (None, None)) + repeats = (gdf["member"] == repeated) & gdf["OSA_ID"].isin( + gdf.loc[gdf["member"] == kept, "OSA_ID"] + ) + gdf = gdf[~repeats].copy() + + names = gdf["C1"] if "C1" in gdf.columns else pd.Series(None, index=gdf.index) + gdf["CT_português"] = names + gdf["CUL_CODIGO"] = self._crop_codes(names) + gdf["CUL_ID"] = gdf["OSA_ID"].fillna(self._island_ids(gdf)) + gdf["OSA_ID"] = gdf["PAR_NUM"].astype("int64") + return gdf + + def _island_ids(self, gdf): + """Stable order: file, PAR_NUM, position, then record order""" + islands = gdf[gdf["OSA_ID"].isna()] + point = islands.geometry.representative_point() + order = pd.DataFrame( + { + "file": islands["member"].map( + {os.path.splitext(m)[0]: i for i, m in enumerate(MEMBERS[self.variant])} + ), + "par": islands["PAR_NUM"].astype("string"), + "x": point.x, + "y": point.y, + } + ).sort_values(["file", "par", "x", "y"], kind="stable") + return pd.Series(range(ISLAND_ID_BASE, ISLAND_ID_BASE + len(order)), index=order.index) + + def _crop_codes(self, names): + keys = names.map( + lambda v: normalise_crop_name(v, keep_question_mark=True), na_action="ignore" + ) + keys = keys.map(lambda k: QUESTION_MARK_ALIASES.get(k, k), na_action="ignore") + keys = keys.replace("", None) + unresolved = sorted({k for k in keys.dropna() if "?" in k}) + assert not unresolved, ( + f"{self.variant}: crop names with an unknown '?': {unresolved}. " + f"Resolve them against pt.csv and add them to QUESTION_MARK_ALIASES." + ) + return keys.map(self._crop_name_lookup()) + + def _crop_name_lookup(self): + # shared with AddHCATMixin, which then does not load it again + if self.hcat_mapping is None: + self.hcat_mapping = load_hcat_mapping(self.hcat_mapping_csv, url=self.mapping_file) + lookup = {} + for entry in self.hcat_mapping: + key = normalise_crop_name(entry["original_name"]) + code = entry["original_code"] + # POUSIO is 089 and 89: the padded code wins + if key not in lookup or code < lookup[key]: + lookup[key] = code + # AZEVEM is 067 (ryegrass) and 076 (lolium), which share the HCAT code + lookup["AZEVEM"] = "067" + return lookup diff --git a/pixi.lock b/pixi.lock index d36dc2e9..11be7cf9 100644 --- a/pixi.lock +++ b/pixi.lock @@ -176,7 +176,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/f7/006b0e81484b5d9f1263f3891d57993f39d4a42206f6e07bd04c6e4ac485/vecorel_cli-0.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/63/bdee5e7ad3c55d61fa1faf6cc5384c4d0e3a583d5e568f248bd1715c8f8e/vecorel_cli-0.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ osx-64: @@ -340,7 +340,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/f7/006b0e81484b5d9f1263f3891d57993f39d4a42206f6e07bd04c6e4ac485/vecorel_cli-0.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/63/bdee5e7ad3c55d61fa1faf6cc5384c4d0e3a583d5e568f248bd1715c8f8e/vecorel_cli-0.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ osx-arm64: @@ -505,7 +505,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/f7/006b0e81484b5d9f1263f3891d57993f39d4a42206f6e07bd04c6e4ac485/vecorel_cli-0.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/63/bdee5e7ad3c55d61fa1faf6cc5384c4d0e3a583d5e568f248bd1715c8f8e/vecorel_cli-0.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ win-64: @@ -668,7 +668,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/f7/006b0e81484b5d9f1263f3891d57993f39d4a42206f6e07bd04c6e4ac485/vecorel_cli-0.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/63/bdee5e7ad3c55d61fa1faf6cc5384c4d0e3a583d5e568f248bd1715c8f8e/vecorel_cli-0.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl - pypi: ./ @@ -823,7 +823,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/f7/006b0e81484b5d9f1263f3891d57993f39d4a42206f6e07bd04c6e4ac485/vecorel_cli-0.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/63/bdee5e7ad3c55d61fa1faf6cc5384c4d0e3a583d5e568f248bd1715c8f8e/vecorel_cli-0.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ osx-64: @@ -962,7 +962,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/f7/006b0e81484b5d9f1263f3891d57993f39d4a42206f6e07bd04c6e4ac485/vecorel_cli-0.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/63/bdee5e7ad3c55d61fa1faf6cc5384c4d0e3a583d5e568f248bd1715c8f8e/vecorel_cli-0.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ osx-arm64: @@ -1102,7 +1102,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/f7/006b0e81484b5d9f1263f3891d57993f39d4a42206f6e07bd04c6e4ac485/vecorel_cli-0.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/63/bdee5e7ad3c55d61fa1faf6cc5384c4d0e3a583d5e568f248bd1715c8f8e/vecorel_cli-0.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ win-64: @@ -1241,7 +1241,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/f7/006b0e81484b5d9f1263f3891d57993f39d4a42206f6e07bd04c6e4ac485/vecorel_cli-0.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/63/bdee5e7ad3c55d61fa1faf6cc5384c4d0e3a583d5e568f248bd1715c8f8e/vecorel_cli-0.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl - pypi: ./ @@ -1358,7 +1358,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/f7/006b0e81484b5d9f1263f3891d57993f39d4a42206f6e07bd04c6e4ac485/vecorel_cli-0.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/63/bdee5e7ad3c55d61fa1faf6cc5384c4d0e3a583d5e568f248bd1715c8f8e/vecorel_cli-0.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ osx-64: @@ -1460,7 +1460,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/f7/006b0e81484b5d9f1263f3891d57993f39d4a42206f6e07bd04c6e4ac485/vecorel_cli-0.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/63/bdee5e7ad3c55d61fa1faf6cc5384c4d0e3a583d5e568f248bd1715c8f8e/vecorel_cli-0.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ osx-arm64: @@ -1563,7 +1563,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/f7/006b0e81484b5d9f1263f3891d57993f39d4a42206f6e07bd04c6e4ac485/vecorel_cli-0.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/63/bdee5e7ad3c55d61fa1faf6cc5384c4d0e3a583d5e568f248bd1715c8f8e/vecorel_cli-0.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ win-64: @@ -1663,7 +1663,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/f7/006b0e81484b5d9f1263f3891d57993f39d4a42206f6e07bd04c6e4ac485/vecorel_cli-0.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/63/bdee5e7ad3c55d61fa1faf6cc5384c4d0e3a583d5e568f248bd1715c8f8e/vecorel_cli-0.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl - pypi: ./ @@ -1807,7 +1807,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/f7/006b0e81484b5d9f1263f3891d57993f39d4a42206f6e07bd04c6e4ac485/vecorel_cli-0.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/63/bdee5e7ad3c55d61fa1faf6cc5384c4d0e3a583d5e568f248bd1715c8f8e/vecorel_cli-0.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ osx-64: @@ -1936,7 +1936,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/f7/006b0e81484b5d9f1263f3891d57993f39d4a42206f6e07bd04c6e4ac485/vecorel_cli-0.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/63/bdee5e7ad3c55d61fa1faf6cc5384c4d0e3a583d5e568f248bd1715c8f8e/vecorel_cli-0.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ osx-arm64: @@ -2066,7 +2066,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/f7/006b0e81484b5d9f1263f3891d57993f39d4a42206f6e07bd04c6e4ac485/vecorel_cli-0.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/63/bdee5e7ad3c55d61fa1faf6cc5384c4d0e3a583d5e568f248bd1715c8f8e/vecorel_cli-0.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: ./ win-64: @@ -2192,7 +2192,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/c7/b0/003792df09decd6849a5e39c28b513c06e84436a54440380862b5aeff25d/tzdata-2025.3-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/f7/006b0e81484b5d9f1263f3891d57993f39d4a42206f6e07bd04c6e4ac485/vecorel_cli-0.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/63/bdee5e7ad3c55d61fa1faf6cc5384c4d0e3a583d5e568f248bd1715c8f8e/vecorel_cli-0.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl - pypi: ./ @@ -2335,7 +2335,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/bc/8737e8d54cf51106118039b83f485a4783112fab49ea9d044b234978a46e/tzdata-2026.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/f7/006b0e81484b5d9f1263f3891d57993f39d4a42206f6e07bd04c6e4ac485/vecorel_cli-0.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/63/bdee5e7ad3c55d61fa1faf6cc5384c4d0e3a583d5e568f248bd1715c8f8e/vecorel_cli-0.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/95/0c/a71501bbc1a674ff72c4d6c2b75f4d9a5af819f5244c3a7558080a8802c5/yarl-1.25.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: ./ @@ -2464,7 +2464,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/bc/8737e8d54cf51106118039b83f485a4783112fab49ea9d044b234978a46e/tzdata-2026.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/f7/006b0e81484b5d9f1263f3891d57993f39d4a42206f6e07bd04c6e4ac485/vecorel_cli-0.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/63/bdee5e7ad3c55d61fa1faf6cc5384c4d0e3a583d5e568f248bd1715c8f8e/vecorel_cli-0.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/55/e6/cd145cff8e5cf60b8b3c41fbecfa2a45028a8dec3fbc52bec03595ff3d3b/yarl-1.25.1-cp311-cp311-macosx_10_9_x86_64.whl - pypi: ./ @@ -2593,7 +2593,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/bc/8737e8d54cf51106118039b83f485a4783112fab49ea9d044b234978a46e/tzdata-2026.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/f7/006b0e81484b5d9f1263f3891d57993f39d4a42206f6e07bd04c6e4ac485/vecorel_cli-0.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/63/bdee5e7ad3c55d61fa1faf6cc5384c4d0e3a583d5e568f248bd1715c8f8e/vecorel_cli-0.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/ce/7c/fbb40fe2d53747c40aa36a9e2bd2178a202f942bda0b670f3306d4aefbce/yarl-1.25.1-cp311-cp311-macosx_11_0_arm64.whl - pypi: ./ @@ -2719,7 +2719,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/bc/8737e8d54cf51106118039b83f485a4783112fab49ea9d044b234978a46e/tzdata-2026.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/f7/006b0e81484b5d9f1263f3891d57993f39d4a42206f6e07bd04c6e4ac485/vecorel_cli-0.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/63/bdee5e7ad3c55d61fa1faf6cc5384c4d0e3a583d5e568f248bd1715c8f8e/vecorel_cli-0.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/aa/9d/8cdceec66a9b940700cb45931741403f045b162afd79bcb93c41cadd0972/yarl-1.25.1-cp311-cp311-win_amd64.whl @@ -2862,7 +2862,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/bc/8737e8d54cf51106118039b83f485a4783112fab49ea9d044b234978a46e/tzdata-2026.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/f7/006b0e81484b5d9f1263f3891d57993f39d4a42206f6e07bd04c6e4ac485/vecorel_cli-0.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/63/bdee5e7ad3c55d61fa1faf6cc5384c4d0e3a583d5e568f248bd1715c8f8e/vecorel_cli-0.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/a2/be/415b00c0fe5a0615b062a456b26623d7ec91c2bee20faea1a14045aa0469/yarl-1.25.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl - pypi: ./ @@ -2992,7 +2992,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/bc/8737e8d54cf51106118039b83f485a4783112fab49ea9d044b234978a46e/tzdata-2026.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/f7/006b0e81484b5d9f1263f3891d57993f39d4a42206f6e07bd04c6e4ac485/vecorel_cli-0.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/63/bdee5e7ad3c55d61fa1faf6cc5384c4d0e3a583d5e568f248bd1715c8f8e/vecorel_cli-0.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/d7/a8/e6c1be0e6761d0f2d10bbf33a3e1e02b99dc83874d92945d7b461a72481e/yarl-1.25.1-cp314-cp314-macosx_10_15_x86_64.whl - pypi: ./ @@ -3122,7 +3122,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/bc/8737e8d54cf51106118039b83f485a4783112fab49ea9d044b234978a46e/tzdata-2026.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/f7/006b0e81484b5d9f1263f3891d57993f39d4a42206f6e07bd04c6e4ac485/vecorel_cli-0.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/63/bdee5e7ad3c55d61fa1faf6cc5384c4d0e3a583d5e568f248bd1715c8f8e/vecorel_cli-0.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/6e/bb/dda344765ffd3430afe1a1c66c866a57fae67786537d4f14607df6505ac1/yarl-1.25.1-cp314-cp314-macosx_11_0_arm64.whl - pypi: ./ @@ -3249,7 +3249,7 @@ environments: - pypi: https://files.pythonhosted.org/packages/24/99/4772b8e00a136f3e01236de33b0efda31ee7077203ba5967fcc76da94d65/texttable-1.7.0-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f9/bc/8737e8d54cf51106118039b83f485a4783112fab49ea9d044b234978a46e/tzdata-2026.4-py2.py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e7/00/3fca040d7cf8a32776d3d81a00c8ee7457e00f80c649f1e4a863c8321ae9/uri_template-1.3.0-py3-none-any.whl - - pypi: https://files.pythonhosted.org/packages/65/f7/006b0e81484b5d9f1263f3891d57993f39d4a42206f6e07bd04c6e4ac485/vecorel_cli-0.3.0-py3-none-any.whl + - pypi: https://files.pythonhosted.org/packages/2c/63/bdee5e7ad3c55d61fa1faf6cc5384c4d0e3a583d5e568f248bd1715c8f8e/vecorel_cli-0.3.1-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/e1/07/c6fe3ad3e685340704d314d765b7912993bcb8dc198f0e7a89382d37974b/win32_setctime-1.2.0-py3-none-any.whl - pypi: https://files.pythonhosted.org/packages/f5/01/349c0386caedbbe488d519f252df54efac8a1459282d466c474bdd84a620/yarl-1.25.1-cp314-cp314-win_amd64.whl @@ -4765,9 +4765,9 @@ packages: - pypi: ./ name: fiboa-cli version: 0.21.0 - sha256: 537013851e61795948896221dda7462beebd711be58e95b5eab43c9542e9f3e0 + sha256: 7d441cb26cb40458ac074fa3f25321db25b084cdea4041d0e4001c081bdda972 requires_dist: - - vecorel-cli==0.3.0 + - vecorel-cli==0.3.1 - beautifulsoup4>=4.12 - spdx-license-list==3.27.0 requires_python: '>=3.11' @@ -13091,10 +13091,10 @@ packages: purls: [] size: 155910 timestamp: 1785359349999 -- pypi: https://files.pythonhosted.org/packages/65/f7/006b0e81484b5d9f1263f3891d57993f39d4a42206f6e07bd04c6e4ac485/vecorel_cli-0.3.0-py3-none-any.whl +- pypi: https://files.pythonhosted.org/packages/2c/63/bdee5e7ad3c55d61fa1faf6cc5384c4d0e3a583d5e568f248bd1715c8f8e/vecorel_cli-0.3.1-py3-none-any.whl name: vecorel-cli - version: 0.3.0 - sha256: 993a78e528f66ed9e52a7dfee4cf94d13a3f6ce7a0cf2a8edba5161f075c456e + version: 0.3.1 + sha256: 853661440f693bc3d3bd31521caf015719059ad1cd65e542282afed9c289f9b5 requires_dist: - pyyaml>=6.0,<7.0 - click>=8.1,<9.0 @@ -13104,6 +13104,7 @@ packages: - numpy>=2.0,<3.0 - pyarrow>=21.0,<24.0 - py7zr>=1.0,<2.0 + - inflate64>=1.0,<2.0 - multivolumefile>=0.2.3,<1.0 - fsspec>=2025.7.0 - jsonschema[format]>=4.20,<5.0 diff --git a/pyproject.toml b/pyproject.toml index 98f2634e..65575af2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ classifiers = [ ] requires-python = ">=3.11" dependencies = [ - "vecorel-cli==0.3.0", + "vecorel-cli==0.3.1", "beautifulsoup4>=4.12", "spdx-license-list==3.27.0", ] diff --git a/tests/data-files/convert/pt/2017.zip b/tests/data-files/convert/pt/2017.zip new file mode 100644 index 00000000..9a054146 Binary files /dev/null and b/tests/data-files/convert/pt/2017.zip differ diff --git a/tests/data-files/convert/pt/2018.zip b/tests/data-files/convert/pt/2018.zip new file mode 100644 index 00000000..19c9c9fa Binary files /dev/null and b/tests/data-files/convert/pt/2018.zip differ diff --git a/tests/data-files/convert/pt/2019.zip b/tests/data-files/convert/pt/2019.zip new file mode 100644 index 00000000..b3384046 Binary files /dev/null and b/tests/data-files/convert/pt/2019.zip differ diff --git a/tests/data-files/convert/pt/2020.zip b/tests/data-files/convert/pt/2020.zip new file mode 100644 index 00000000..d9a22174 Binary files /dev/null and b/tests/data-files/convert/pt/2020.zip differ diff --git a/tests/data-files/convert/pt/2021.zip b/tests/data-files/convert/pt/2021.zip new file mode 100644 index 00000000..23a93d90 Binary files /dev/null and b/tests/data-files/convert/pt/2021.zip differ diff --git a/tests/data-files/convert/pt/2022.zip b/tests/data-files/convert/pt/2022.zip new file mode 100644 index 00000000..c24b748d Binary files /dev/null and b/tests/data-files/convert/pt/2022.zip differ diff --git a/tests/data-files/convert/pt/README.md b/tests/data-files/convert/pt/README.md index 785caaa0..4feebdb7 100644 --- a/tests/data-files/convert/pt/README.md +++ b/tests/data-files/convert/pt/README.md @@ -17,3 +17,59 @@ done `Culturas` (empty) and `Codes` (the NUTS 3 code list) are kept in the 2025 file so the layer filter is exercised on the layers it has to skip. + +2020-2022 ship an archive of regional files instead of one GeoPackage, so those fixtures are +zips. Each keeps 100 features per region, plus a few features of a layer the edition's filter +has to skip (`Parcelas*`, `Baldios*`), and 2020/2021 keep a slice of the campaign's crop table +so the join is exercised: enough rows to match most of the sampled fields, a few keys that +match none of them, and — for 2021 — the fourteen rows keyed on `Osa_id = 0` that make the +crop table non-unique. + +`Culturas_2021.dbf` is deliberately *not* in the 2021 member list. It has no geometry, and +`DATA_LAYER` matches its name, so a converter that filtered layers by that pattern alone would +convert a 3.4M-row attribute table as if it were field boundaries. + +``` +for f in Ocupacoes_solo_ALENTEJO Ocupacoes_solo_ALGARVE ... ; do + ogr2ogr $f.shp $DOWNLOADED_SOURCE/$f.shp -limit 100 +done +ogr2ogr -f "ESRI Shapefile" -nlt NONE Culturas_2021.dbf $DOWNLOADED_SOURCE/Culturas_2021.dbf \ + -where "Osa_id IN (, , 0)" +zip -9 2021.zip *.shp *.shx *.dbf *.prj *.cpg # plain deflate, see below +``` + +## 2017-2019 + +Same shape as the 2020-2022 fixtures (a zip of regional files), but these editions are +plain shapefiles with one layer each, so the member list is the only thing that decides +what gets read. All ten members of each edition must be present or `get_data` fails on +the missing path, so the uninteresting regions are cut to 20 rows and the budget spent on +the ones that carry a mechanism: + +- **2018 `Ocupacoes_solo_Norte_N` and `Ocupacoes_solo_Norte_S`** keep 100 rows each, of + which **30 share an OSA_ID**. The real edition repeats 154,980 fields between these two + members, byte for byte, and converting both as published inflates it by that many rows. + The unique halves deliberately exclude every other shared id, so the fixture overlap is + exactly 30 and `test_2018_drops_exactly_the_duplicated_rows` can assert the count. +- **2017 `Ocupacoes_solo_RAA` and `Ocupacoes_solo_RAM`** publish no `OSA_ID` at all and + repeat `PAR_NUM`, so the fixtures are chosen to keep rows whose `PAR_NUM` repeats. They + exercise the synthesised id on the condition that forced it. +- **2018 `ocupacoes.solo.Norte_N1.2018jun10`** is awkward three ways at once and carries + all fourteen crop names in which the provider lost an accent to a literal `?`. It is + also the only member with lowercase, non-contiguous crop columns (`c1..c7`, `c9`, no + `c8`) and one of two published in EPSG:3763. Note the dots inside the filename. +- **2019 `Ocupacoes_solo_RAA`** keeps 100 rows for its 28 crop columns, and + **`ocupacoes_solo_n_1`** is the EPSG:3763 case for that year. +- **`Parcelas_*`** is in each zip but in no member list: it is the parcel block geometry, + not field boundaries. 2019 also keeps an orphan `osas_az_ocidental.qpj` and one of the + `*.sr.lock` files the provider shipped, so the fixture proves the junk is simply never + named rather than actively skipped. + +The rows were selected from the real archives by these criteria rather than with a plain +`ogr2ogr -limit`, so a regenerated fixture has to keep them for the tests to hold. + +## Compression + +The real 2017-2022 archives are Deflate64, which python's `zipfile` cannot read. The 2022 +fixture is Deflate64 too, recompressed with `python to_deflate64.py 2022.zip`, so the tests +extract the format IFAP publishes; the other fixtures are plain Deflate. diff --git a/tests/data-files/convert/pt/to_deflate64.py b/tests/data-files/convert/pt/to_deflate64.py new file mode 100644 index 00000000..89bb98d9 --- /dev/null +++ b/tests/data-files/convert/pt/to_deflate64.py @@ -0,0 +1,51 @@ +"""Recompress a fixture archive with Deflate64, as IFAP publishes 2017-2022. + +python's zipfile can read and write neither, so its compressor lookup is extended with +inflate64 for the duration of the rewrite. Usage: python to_deflate64.py 2022.zip +""" + +import sys +import zipfile +from unittest.mock import patch + +import inflate64 + +ZIP_DEFLATED64 = 9 + + +class Deflate64Compressor: + def __init__(self): + self._deflater = inflate64.Deflater() + + def compress(self, data): + return self._deflater.deflate(data) + + def flush(self): + return self._deflater.flush() + + +def get_compressor(compress_type, compresslevel=None): + if compress_type == ZIP_DEFLATED64: + return Deflate64Compressor() + return original_get_compressor(compress_type, compresslevel) + + +def check_compression(compression): + if compression != ZIP_DEFLATED64: + original_check_compression(compression) + + +original_get_compressor = zipfile._get_compressor +original_check_compression = zipfile._check_compression + +for path in sys.argv[1:]: + with zipfile.ZipFile(path) as source: + members = [(info, source.read(info)) for info in source.infolist()] + with ( + patch.object(zipfile, "_get_compressor", get_compressor), + patch.object(zipfile, "_check_compression", check_compression), + zipfile.ZipFile(path, "w") as target, + ): + for info, data in members: + info.compress_type = ZIP_DEFLATED64 + target.writestr(info, data) diff --git a/tests/test_convert.py b/tests/test_convert.py index e30da232..fab8fbcc 100644 --- a/tests/test_convert.py +++ b/tests/test_convert.py @@ -1,6 +1,7 @@ import json import re import sys +from contextlib import ExitStack from csv import DictReader from os.path import exists from unittest.mock import patch @@ -18,6 +19,17 @@ Optionally use `-lco ENCODING=UTF-8` if you have character encoding issues. """ +# PT editions besides the default one, 2023 +PT_EDITIONS = ("2025", "2022", "2021", "2020", "2019", "2018", "2017") +PT_COLUMNS = ( + "determination:datetime", + "metrics:area", + "metrics:perimeter", + "crop:code", + "block_id", + "id", +) + tests = [ "at", "at_block", @@ -38,7 +50,7 @@ "nl", "nl_block", "pt", - "pt#2025", + *(f"pt#{year}" for year in PT_EDITIONS), "dk", "dk#2008", "be_wal", @@ -101,6 +113,8 @@ def _input_files(converter, *names): "ai4sf": _input_files("ai4sf", "1_vietnam_areas.gpkg", "4_cambodia_areas.gpkg"), "nl": {"variant": "2023"}, "dk#2008": {"variant": "2008"}, + "pt": {"variant": "2023"}, + **{f"pt#{year}": {"variant": year} for year in PT_EDITIONS}, # the fixture archive holds the 2024 edition only; the published one holds both "lt": {"variant": "2024"}, # the fixture is the 2023 file; the converter's default is the newest edition @@ -203,6 +217,17 @@ def _input_files(converter, *names): "ie_lpis#2025": ("determination:datetime", "metrics:area", "crop:code", "id"), # derived from the archive date in file_migration() "pl_block": ("determination:datetime",), + # only 2017-2019 and 2023 publish a crop name + "pt": (*PT_COLUMNS, "crop:name"), + **{ + f"pt#{year}": (*PT_COLUMNS, "crop:name") if year <= "2019" else PT_COLUMNS + for year in PT_EDITIONS + }, +} + +# Mapping loaders to patch besides commons.hcat, e.g. where a converter imports one by name +mapping_lookups = { + "pt": ("fiboa_cli.datasets.pt.load_hcat_mapping",), } @@ -229,7 +254,10 @@ def load_mapping(csv_file=None, url=None): path = f"tests/data-files/convert/{converter_id}" kwargs = extra_convert_parameters.get(converter, {}) - ConvertData(converter_id).convert(target=tmp_parquet_file, cache=path, **kwargs) + with ExitStack() as stack: + for target in mapping_lookups.get(converter_id, ()): + stack.enter_context(patch(target, side_effect=load_mapping)) + ConvertData(converter_id).convert(target=tmp_parquet_file, cache=path, **kwargs) out, err = capsys.readouterr() output = out + err @@ -253,6 +281,14 @@ def load_mapping(csv_file=None, url=None): f"collection metadata. Produced columns: {sorted(df.columns)}" ) + # a float id stringifies as "2315738.0": unique, valid and wrong + if required and "id" in df.columns: + floaty = df["id"].astype("string").str.fullmatch(r"-?\d+\.0*").fillna(False) + assert not floaty.any(), ( + f"{converter}: {int(floaty.sum()):,} id(s) are stringified floats, " + f"e.g. {df.loc[floaty, 'id'].head(3).tolist()}" + ) + if "metrics:area" in df.columns and converter not in ("de_bb",): # Check for accidental hectare conversion; fields should be more than 10 square meters assert (df["metrics:area"] > 10).all() diff --git a/tests/test_convert_pt.py b/tests/test_convert_pt.py new file mode 100644 index 00000000..02cbdd20 --- /dev/null +++ b/tests/test_convert_pt.py @@ -0,0 +1,225 @@ +"""Checks for PT 2017-2022 that the column expectations in test_convert.py cannot see.""" + +import glob +import os +import sys +import zipfile +from csv import DictReader +from unittest.mock import patch + +import pandas as pd +import pyarrow.parquet as pq +import pyogrio +from loguru import logger +from pytest import fixture, mark, raises + +from fiboa_cli.convert import ConvertData +from fiboa_cli.datasets import pt + +PT = "tests/data-files/convert/pt" +FIXTURE_ROWS = {"2017": 480, "2018": 385, "2019": 360} +# fields the 2018 fixture has in both Norte_N and Norte_S +DUPLICATED_IN_2018 = 30 + + +def _load_mapping(csv_file=None, url=None): + if csv_file and "://" in csv_file: + csv_file = csv_file.split("/")[-1] + path = url if url and "://" not in url else f"{PT}/{csv_file}" + return list(DictReader(open(path, "r", encoding="utf-8"))) + + +def _convert(target, variant, converter=None, **kwargs): + # loguru sinks are global, so the sink must not outlive the call + sink = logger.add(sys.stdout, format="{message}", level="DEBUG", colorize=False) + try: + # pt.py imports load_hcat_mapping by name, so it is patched in both modules + with ( + patch("fiboa_cli.datasets.commons.hcat.load_hcat_mapping", side_effect=_load_mapping), + patch("fiboa_cli.datasets.pt.load_hcat_mapping", side_effect=_load_mapping), + ): + kwargs.setdefault("cache", PT) + if converter is None: + ConvertData("pt").convert(target=target, variant=variant, **kwargs) + else: + converter.convert(target, variant=variant, **kwargs) + finally: + logger.remove(sink) + return pq.read_table(target).to_pandas() + + +def _island_ids(df): + ids = df["id"].astype("int64") + return ids[ids >= pt.ISLAND_ID_BASE] + + +@fixture(scope="module") +def converted(tmp_path_factory): + out = {} + for variant in ("2017", "2018", "2019"): + target = tmp_path_factory.mktemp(variant) / "test.parquet" + out[variant] = _convert(target, variant) + return out + + +@mark.parametrize("variant", ("2017", "2018", "2019")) +def test_ids_are_unique_across_the_edition(converted, variant): + df = converted[variant] + # repeated ids would get a ~ suffix and show only as extra rows + assert len(df) == FIXTURE_ROWS[variant] + assert df["id"].is_unique + assert not df["id"].astype("string").str.contains("~").any() + assert df["metrics:area"].notna().all() + + +def test_2018_drops_exactly_the_duplicated_rows(converted): + rows_read = sum( + pyogrio.read_info(f"{PT}/extracted.2018/{m}")["features"] for m in pt.MEMBERS["2018"] + ) + assert rows_read - len(converted["2018"]) == DUPLICATED_IN_2018 + + +def test_an_undeclared_member_overlap_shows_as_suffixed_ids(tmp_path): + with patch.dict(pt.OVERLAPPING_MEMBERS, {"2018": (None, None)}): + df = _convert(tmp_path / "x.parquet", "2018") + # both copies of each repeated id get a suffix + assert int(df["id"].astype("string").str.contains("~").sum()) == 2 * DUPLICATED_IN_2018 + + +def test_2017_island_rows_survive_without_a_source_identifier(converted): + df = converted["2017"] + islands = sum( + pyogrio.read_info(f"{PT}/extracted.2017/Ocupacoes_solo_{r}.shp")["features"] + for r in ("RAA", "RAM") + ) + synthesised = df["id"].astype("int64") >= pt.ISLAND_ID_BASE + assert int(synthesised.sum()) == islands + assert df.loc[synthesised, "id"].is_unique + assert df.loc[~synthesised, "id"].astype("int64").max() < pt.ISLAND_ID_BASE + assert (df.loc[synthesised, "crop:code"] == "").all() + + +def test_2017_island_ids_are_stable_across_runs(tmp_path): + key = ["block_id", "metrics:area"] + runs = [_convert(tmp_path / f"{n}.parquet", "2017") for n in ("a", "b")] + a, b = (df[df["id"].astype("int64") >= pt.ISLAND_ID_BASE].sort_values(key) for df in runs) + assert a["id"].tolist() == b["id"].tolist() + + +def test_2017_island_ids_follow_the_declared_sort(converted): + islands = converted["2017"].loc[_island_ids(converted["2017"]).index].copy() + islands["n"] = islands["id"].astype("int64") - pt.ISLAND_ID_BASE + # RAA is numbered first, then RAM; within a file the ids ascend with PAR_NUM + start = 0 + for region in ("RAA", "RAM"): + rows = pyogrio.read_info(f"{PT}/extracted.2017/Ocupacoes_solo_{region}.shp")["features"] + block = islands[(islands["n"] >= start) & (islands["n"] < start + rows)] + assert len(block) == rows + assert block.sort_values("n")["block_id"].is_monotonic_increasing + start += rows + + +@mark.parametrize( + "variant,member", + ( + ("2017", "ocupacoes_solo_norte_n1.shp"), + ("2018", "ocupacoes.solo.Norte_N1.2018jun10.shp"), + ("2019", "ocupacoes_solo_n_1.shp"), + ), +) +def test_projected_members_still_get_an_area(converted, variant, member): + """The only files published in EPSG:3763; the area is measured after reprojection.""" + assert pyogrio.read_info(f"{PT}/extracted.{variant}/{member}")["crs"] == "EPSG:3763" + area = converted[variant]["metrics:area"] + assert area.notna().all() + # square metres, not hectares or degrees + assert (area > 10).all() + assert area.median() > 100 + + +def test_unresolved_question_mark_raises(): + converter = pt.PTConverter() + converter.variant = "2018" + converter.hcat_mapping = _load_mapping("pt.csv") + with raises(AssertionError, match=r"unknown '\?'"): + converter._crop_codes(pd.Series(["BATATA", "COURG?TTE"])) + + +def test_crop_names_resolve_across_the_spellings_2017_uses(converted): + pairs = converted["2017"][["crop:name", "crop:code"]].drop_duplicates() + coded = pairs[pairs["crop:code"] != ""] + assert len(coded[coded["crop:name"].str.contains("_", na=False)]) >= 5 + assert len(coded[coded["crop:name"].str.contains("[ÁÂÃÇÉÊÍÓÔÕÚ]", na=False)]) >= 3 + lookup = dict(zip(coded["crop:name"], coded["crop:code"])) + assert lookup["PRADOS_TEMPORARIOS"] == lookup["PRADOS TEMPORÁRIOS"] + + +def test_2018_question_mark_names_survive_the_whole_pipeline(converted): + """They are all in Norte_N1, whose crop columns are lowercase and skip c8.""" + df = converted["2018"] + q = df[df["crop:name"].str.contains(r"\?", na=False)] + assert len(q) > 0 + assert (q["crop:code"] != "").all() + assert q["hcat:code"].notna().all() + + +@mark.parametrize("variant", ("2017", "2018", "2019")) +def test_block_id_is_an_integer(converted, variant): + df = converted[variant] + assert str(df["block_id"].dtype) in ("Int64", "int64") + assert df["block_id"].min() > 10**11 + + +def test_2022_extracts_the_deflate64_archive(tmp_path): + """The published 2017-2022 archives are Deflate64, and so is this fixture.""" + archive = f"{PT}/2022.zip" + with zipfile.ZipFile(archive) as zf: + assert {info.compress_type for info in zf.infolist()} == {9} + # an empty cache, so the archive is extracted + df = _convert( + tmp_path / "out.parquet", + "2022", + cache=str(tmp_path), + input_files={archive: pt.MEMBERS["2022"]}, + ) + assert len(df) == 400 + + +def _fixture_crop_codes(year, folder): + """Each field's crop code, looked up in the crop table without the converter.""" + with zipfile.ZipFile(f"{PT}/{year}.zip") as zf: + zf.extractall(folder) + ids = pd.concat( + pyogrio.read_dataframe(path, columns=["OSA_ID"], read_geometry=False)["OSA_ID"] + for pattern in pt.MEMBERS[year] + for path in glob.glob(os.path.join(folder, "**", pattern), recursive=True) + ).astype("int64") + (table,) = glob.glob(os.path.join(folder, "**", pt.CROP_TABLE[year]), recursive=True) + key = next(f for f in pyogrio.read_info(table)["fields"] if f.lower() == "osa_id") + crops = pyogrio.read_dataframe(table, columns=[key, "C1"], read_geometry=False) + crops = crops[crops[key] != 0] + codes = ids.map(crops.set_index(crops[key].astype("int64"))["C1"]).fillna("") + return dict(zip(ids, codes)) + + +@mark.parametrize("year", ("2020", "2021")) +def test_crop_codes_are_joined_from_the_crop_table(tmp_path, year): + expected = _fixture_crop_codes(year, tmp_path / "source") + assert any(expected.values()) and not all(expected.values()) + df = _convert(tmp_path / "out.parquet", year) + assert dict(zip(df["id"].astype("int64"), df["crop:code"])) == expected + + +def test_a_reused_converter_starts_each_edition_afresh(tmp_path): + converter = pt.PTConverter() + _convert(tmp_path / "2020.parquet", "2020", converter=converter) + reused = _convert(tmp_path / "2021.parquet", "2021", converter=converter) + fresh = _convert(tmp_path / "2021_fresh.parquet", "2021") + key = ["id", "crop:code"] + assert reused[key].sort_values("id").values.tolist() == ( + fresh[key].sort_values("id").values.tolist() + ) + + first = _convert(tmp_path / "a.parquet", "2017", converter=converter) + second = _convert(tmp_path / "b.parquet", "2017", converter=converter) + assert sorted(_island_ids(first)) == sorted(_island_ids(second))