Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions tests/test_selection_guard.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,3 +39,22 @@ def test_guard_type_is_disjoint_from_contract_vocabulary():
"sampled_forecast_sidecar",
):
assert f'"type": "{contract_type}"' not in _SRC


def test_compact_description_fits_the_store_limit():
"""Run-0 lesson (2026-07-27): Appwrite's description attribute caps at 255
chars; the old prose-prefixed provenance exceeded it and stranded the
historical upload as an orphan file without a metadata document."""
from views_postprocessing.delivery import provenance
from views_postprocessing.delivery.provenance import DESCRIPTION_MAX, compact_description

prov = provenance.build_provenance(
lookup_version="land_gaul@272cdb01",
region="land_gaul",
expected_cell_count=64742,
actual_cell_count=64742,
unmapped_count=0,
)
text = compact_description(prov)
assert len(text) <= DESCRIPTION_MAX
assert "land_gaul@272cdb01" in text # provenance survives, prose does not
28 changes: 28 additions & 0 deletions views_postprocessing/delivery/provenance.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,3 +52,31 @@ def build_provenance(
if fill_count is not None:
provenance["fill_count"] = fill_count
return provenance


DESCRIPTION_MAX = 255 # Appwrite metadata attribute limit (run-0 lesson, 2026-07-27)


def compact_description(prov: dict) -> str:
"""The provenance dict as compact JSON, guaranteed to fit the store's
255-char description attribute — the only structured carrier the upload
exposes (C-15). The decorative prose prefix died here: it cost the run-0
historical document (metadata rejected; file stranded as an invisible
orphan). Fails loud if even compact JSON cannot fit — never truncates
silently."""
import json

text = json.dumps(prov, separators=(",", ":"))
if len(text) > DESCRIPTION_MAX:
essential = {
k: prov[k]
for k in ("lookup_version", "region", "expected_cell_count", "actual_cell_count", "unmapped_count")
if k in prov
}
text = json.dumps(essential, separators=(",", ":"))
if len(text) > DESCRIPTION_MAX:
raise ValueError(
f"provenance description cannot fit the store's {DESCRIPTION_MAX}-char "
f"limit even compacted ({len(text)} chars) — refusing to truncate silently."
)
return text
27 changes: 15 additions & 12 deletions views_postprocessing/unfao/managers/unfao.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
from views_pipeline_core.managers.ensemble import EnsemblePathManager
import pandas as pd
import io
import json
from datetime import datetime
import os
from dotenv import load_dotenv
Expand Down Expand Up @@ -55,7 +54,7 @@ def download(self, file_id: str) -> bytes:
)

def upload(self, file_path, *, filename, name, doc_type, category, loa, targets, description=None) -> None:
self._dsm.upload_data(
result = self._dsm.upload_data(
file=file_path,
filename=filename,
name=name,
Expand All @@ -65,6 +64,18 @@ def upload(self, file_path, *, filename, name, doc_type, category, loa, targets,
targets=targets,
description=description,
)
# The store module degrades gracefully (its ADR-046 policy) and only LOGS
# metadata failures — which strands an invisible orphan file (run-0
# historical, 2026-07-27). The delivery fails loud instead.
success = getattr(result, "success", None)
if success is None and hasattr(result, "to_dict"):
success = result.to_dict().get("success")
if success is False:
error = getattr(result, "error", None) or "unknown store error"
raise RuntimeError(
f"upload of {filename!r} did not fully succeed (file may be an "
f"orphan without a metadata document): {error}"
)


class UNFAOPostProcessorManager(PostprocessorManager, ForecastingModelManager):
Expand Down Expand Up @@ -582,11 +593,7 @@ def _historical_frame_description(self, table, timestamp: str) -> str:
actual_cell_count=len(frame_extraction.cells_of(self._historical_frame)),
unmapped_count=historical.unmapped_cell_count(table),
)
return (
f"Enriched with geographic metadata on {timestamp} using precomputed GAUL "
f"lookup (ADR-011, version={self._enricher.lookup_version}). "
f"provenance={json.dumps(prov, separators=(',', ':'))}"
)
return provenance.compact_description(prov)

def _build_historical_artifact(self, directory) -> tuple:
"""Frame-built historical artifact staged into ``directory``; returns
Expand Down Expand Up @@ -619,8 +626,4 @@ def _delivery_description(self, df: pd.DataFrame, timestamp: str) -> str:
actual_cell_count=len(extraction.cells_of(df)),
unmapped_count=extraction.unmapped_cell_count(df, METADATA_COLS),
)
return (
f"Enriched with geographic metadata on {timestamp} using precomputed GAUL "
f"lookup (ADR-011, version={self._enricher.lookup_version}). "
f"provenance={json.dumps(prov, separators=(',', ':'))}"
)
return provenance.compact_description(prov)
Loading