diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f1608c2..9afcf0f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,7 @@ on: - feat/ar1-metadata-fabric-active-metadata-projection-execution - feat/ar1-metadata-fabric-active-metadata-binding-reconciliation - feat/ar1-metadata-fabric-durable-active-metadata-promotion + - feat/ar1-metadata-fabric-object-store-active-metadata-promotion env: PYTHON_VERSION: "3.13" @@ -196,6 +197,9 @@ jobs: - name: Validate metadata fabric object-store Active Metadata promotion evidence run: python -m data_agent.metadata_fabric_object_store_active_metadata_promotion validate + - name: Validate metadata fabric real-feature ingestion evidence + run: python -m data_agent.metadata_fabric_real_feature_ingestion validate + - name: Validate Active Metadata consumer deployment boundary run: python -m data_agent.active_metadata_consumer_deployment validate @@ -286,6 +290,7 @@ jobs: data_agent/test_metadata_fabric_active_metadata_binding_reconciliation.py \ data_agent/test_metadata_fabric_durable_active_metadata_promotion.py \ data_agent/test_metadata_fabric_object_store_active_metadata_promotion.py \ + data_agent/test_metadata_fabric_real_feature_ingestion.py \ data_agent/test_metadata_fabric_lineage_delivery.py \ data_agent/test_metadata_fabric_provider_identity.py \ data_agent/test_metadata_fabric_gravitino_identity.py \ diff --git a/config/metadata-fabric-real-feature-ingestion.local.yaml b/config/metadata-fabric-real-feature-ingestion.local.yaml new file mode 100644 index 00000000..e55b5727 --- /dev/null +++ b/config/metadata-fabric-real-feature-ingestion.local.yaml @@ -0,0 +1,60 @@ +schema: gda.real_feature_ingestion_profile.v1 +environment: local_docker_desktop + +dependencies: + m321_profile_path: config/metadata-fabric-object-store-active-metadata-promotion.local.yaml + m321_evidence_path: docs/evidence/metadata-fabric-object-store-active-metadata-promotion-2026-07-31.json + m321_evidence_sha256: d73754c53cf16d888aa345baa5d079cc7fd98d8b84db747f52188c1a69bf1628 + m310_profile_path: config/metadata-fabric-spark-object-store-interoperability.local.yaml + m310_evidence_path: docs/evidence/metadata-fabric-spark-object-store-interoperability-2026-07-29.json + m310_evidence_fingerprint: 05844457efb378581fb7fc2e7ed3c706819b2d8fa5a52b2f82577051d38c2cd1 + +source: + source_label: chongqing-central-cultural-districts + identifier_field: Bsm + expected_feature_count: 20 + expected_srid: 4490 + expected_geometry_dimension: 3 + expected_geometry_family: polygon + +target: + metalake: gda_chongqing_m3_22 + catalog: lakehouse + schema: cultural_heritage + table: cultural_districts + catalog_type: RELATIONAL + catalog_provider: lakehouse-iceberg + catalog_backend: jdbc + uri: jdbc:postgresql://gravitino-persistence-postgresql:5432/iceberg + warehouse: s3://gda-metadata-warehouse/warehouse + jdbc_driver: org.postgresql.Driver + io_impl: org.apache.iceberg.aws.s3.S3FileIO + s3_endpoint: http://metadata-object-store:9000 + s3_region: us-east-1 + s3_path_style_access: true + bucket: gda-metadata-warehouse + object_prefix: warehouse/cultural_heritage/cultural_districts/ + +identity: + service_admin: gda-object-store-admin + user: gda-object-store-active-metadata-promoter + role: gda-object-store-cultural-district-projector + material_delivery: runtime_generated_ephemeral_kubernetes_object + +authorization: + policy_version_ref: policy://gda/metadata-fabric/real-feature-ingestion/v1 + evaluator_subject: workload:real-feature-ingestion-policy-evaluator + approver_subject: human:metadata-platform-owner + approval_reason: Approve one content-bound local Spark/Sedona ingestion rehearsal. + +claims: + predecessor_history_changed: false + ingestion_persisted_to_gda_control: false + protected_workload_identity_verified: false + durable_catalog_verified: false + production_object_store_verified: false + oidc_verified: false + tls_verified: false + flink_conformance_verified: false + production_ingestion_verified: false + production_ready: false diff --git a/data_agent/metadata_fabric_real_feature_ingestion.py b/data_agent/metadata_fabric_real_feature_ingestion.py new file mode 100644 index 00000000..07e08955 --- /dev/null +++ b/data_agent/metadata_fabric_real_feature_ingestion.py @@ -0,0 +1,1815 @@ +"""Ingest a bounded real Chongqing feature slice into JDBC/S3 Iceberg. + +M3-22 consumes the checked M3-21 runtime-bound promotion, reads the same local +Shapefile through an explicit CLI path, and creates a path-free row-set binding. +One authorized Spark/Sedona Job writes the rows and immediately replays the +same plan as a readback-proven no-op. Source paths, feature payloads and runtime +credentials never enter the committed evidence. + +The result remains local evidence. It does not persist the output candidate to +GDA Control or establish protected identity, production object storage, full +engine conformance, production ingestion or production readiness. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import secrets +import time +from collections.abc import Mapping +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any, Literal +from uuid import UUID, uuid5 + +import geopandas as gpd +import yaml +from pydantic import BaseModel, ConfigDict, Field, SecretStr, model_validator + +from . import metadata_fabric_durable_active_metadata_promotion as durable +from . import metadata_fabric_ingestion_replay as replay +from . import metadata_fabric_object_store_active_metadata_promotion as m321 +from . import metadata_fabric_spark_object_store_interoperability as m310 +from .platform_authorization import ( + build_approval_artifact, + build_policy_decision_artifact, + validate_run_authorization_evidence, +) +from .platform_contracts import ( + ApprovalRecord, + Artifact, + ArtifactRole, + LineageEvent, + LineageEventType, + PlatformRun, + PolicyDecision, + QualityResult, + QualityVerdict, + ResourceVersion, + RunPolicyReferences, + SubjectContext, + canonical_json_bytes, + canonical_json_fingerprint, + quality_result_fingerprint, +) +from .spatial_dataset_bundle import ( + build_shapefile_bundle_inventory, + validate_shapefile_bundle_inventory, +) + +PROFILE_SCHEMA = "gda.real_feature_ingestion_profile.v1" +PLAN_SCHEMA = "gda.real_feature_ingestion_plan.v1" +ROW_SET_SCHEMA = "gda.real_feature_row_set.v1" +CONTRACT_SCHEMA = "gda.real_feature_ingestion_contract.v1" +OBSERVATION_SCHEMA = "gda.real_feature_ingestion_observation.v1" +EVIDENCE_SCHEMA = "gda.real_feature_ingestion_evidence.v1" +VALIDATION_SCHEMA = "gda.real_feature_ingestion_validation.v1" +PROBE_RESULT_SCHEMA = "gda.real_feature_ingestion_probe_result.v1" +ACTION = "metadata_fabric.ingest_real_feature_slice" +TENANT = m321.TENANT +SOURCE_RESOURCE_VERSION_ID = m321.RESOURCE_VERSION_ID +OUTPUT_RESOURCE_VERSION_ID = UUID("a6000000-0000-4000-8000-000000000002") +DEFINITION_VERSION_ID = UUID("a9000000-0000-4000-8000-000000000008") +RUN_ID = UUID("a9000000-0000-4000-8000-000000000009") +OUTPUT_RESOURCE_URN = ( + "gda://metadata-authorization-local/data_product/chongqing-cultural-districts-iceberg" +) +WORKLOAD = "workload:real-feature-ingestion-executor" +QUALITY_EVALUATOR = "workload:real-feature-spatial-quality-evaluator" +M321_EVIDENCE_SHA256 = ( + "d73754c53cf16d888aa345baa5d079cc7fd98d8b84db747f52188c1a69bf1628" +) +M310_EVIDENCE_FINGERPRINT = m321.M310_EVIDENCE_FINGERPRINT + +REPO_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_PROFILE_PATH = REPO_ROOT / "config/metadata-fabric-real-feature-ingestion.local.yaml" +DEFAULT_EVIDENCE_PATH = ( + REPO_ROOT / "docs/evidence/metadata-fabric-real-feature-ingestion-2026-07-31.json" +) +DEFAULT_WRAPPER_PATH = REPO_ROOT / "scripts/metadata-fabric-real-feature-ingestion.sh" +DEFAULT_JOB_PATH = REPO_ROOT / "k8s/metadata-fabric-real-feature-ingestion/spark-job.yaml" + +GRAVITINO_COLUMNS = ( + {"name": "BSM", "type": "string", "nullable": False}, + {"name": "geometry", "type": "binary", "nullable": False}, + {"name": "srid", "type": "integer", "nullable": False}, + {"name": "min_x", "type": "double", "nullable": False}, + {"name": "min_y", "type": "double", "nullable": False}, + {"name": "max_x", "type": "double", "nullable": False}, + {"name": "max_y", "type": "double", "nullable": False}, + {"name": "row_sha256", "type": "string", "nullable": False}, +) +ICEBERG_FIELDS = ( + {"name": "BSM", "required": True, "type": "string"}, + {"name": "geometry", "required": True, "type": "binary"}, + {"name": "srid", "required": True, "type": "int"}, + {"name": "min_x", "required": True, "type": "double"}, + {"name": "min_y", "required": True, "type": "double"}, + {"name": "max_x", "required": True, "type": "double"}, + {"name": "max_y", "required": True, "type": "double"}, + {"name": "row_sha256", "required": True, "type": "string"}, +) +SPARK_COLUMNS = tuple(item["name"] for item in ICEBERG_FIELDS) +FALSE_CLAIMS = ( + "predecessor_history_changed", + "ingestion_persisted_to_gda_control", + "source_dataset_committed", + "source_absolute_path_committed", + "source_feature_payload_committed", + "protected_workload_identity_verified", + "durable_catalog_verified", + "production_object_store_verified", + "oidc_verified", + "tls_verified", + "flink_conformance_verified", + "spark_conformance_verified", + "production_ingestion_verified", + "platform_run_succeeded", + "production_ready", +) + + +class RealFeatureIngestionError(RuntimeError): + """The real feature ingestion contract failed closed.""" + + +class _FrozenModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + +class DependencyProfile(_FrozenModel): + m321_profile_path: str + m321_evidence_path: str + m321_evidence_sha256: Literal[M321_EVIDENCE_SHA256] + m310_profile_path: str + m310_evidence_path: str + m310_evidence_fingerprint: Literal[M310_EVIDENCE_FINGERPRINT] + + +class SourceProfile(_FrozenModel): + source_label: Literal["chongqing-central-cultural-districts"] + identifier_field: Literal["Bsm"] + expected_feature_count: Literal[20] + expected_srid: Literal[4490] + expected_geometry_dimension: Literal[3] + expected_geometry_family: Literal["polygon"] + + +class ClaimProfile(_FrozenModel): + predecessor_history_changed: Literal[False] + ingestion_persisted_to_gda_control: Literal[False] + protected_workload_identity_verified: Literal[False] + durable_catalog_verified: Literal[False] + production_object_store_verified: Literal[False] + oidc_verified: Literal[False] + tls_verified: Literal[False] + flink_conformance_verified: Literal[False] + production_ingestion_verified: Literal[False] + production_ready: Literal[False] + + +class AuthorizationProfile(_FrozenModel): + policy_version_ref: Literal["policy://gda/metadata-fabric/real-feature-ingestion/v1"] + evaluator_subject: Literal["workload:real-feature-ingestion-policy-evaluator"] + approver_subject: Literal["human:metadata-platform-owner"] + approval_reason: str + + +class RealFeatureIngestionProfile(_FrozenModel): + profile_schema: Literal[PROFILE_SCHEMA] = Field(alias="schema") + environment: Literal["local_docker_desktop"] + dependencies: DependencyProfile + source: SourceProfile + target: m321.ObjectStoreTarget + identity: m321.IdentityProfile + authorization: AuthorizationProfile + claims: ClaimProfile + + +def _mapping(value: Any) -> Mapping[str, Any]: + return value if isinstance(value, Mapping) else {} + + +def _list(value: Any) -> list[Any]: + return value if isinstance(value, list) else [] + + +def _resolve_repo_path(value: str) -> Path: + path = Path(value) + candidate = path if path.is_absolute() else REPO_ROOT / path + try: + resolved = candidate.resolve(strict=True) + resolved.relative_to(REPO_ROOT) + except (OSError, ValueError) as exc: + raise RealFeatureIngestionError("dependency path escapes the repository") from exc + return resolved + + +def _load_json_object(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError("JSON document must be an object") + return value + + +def _file_record(path: Path) -> dict[str, Any]: + payload = path.read_bytes() + return { + "path": str(path.relative_to(REPO_ROOT)), + "size_bytes": len(payload), + "sha256": hashlib.sha256(payload).hexdigest(), + } + + +def load_profile(path: Path = DEFAULT_PROFILE_PATH) -> RealFeatureIngestionProfile: + try: + value = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError("profile must be an object") + profile = RealFeatureIngestionProfile.model_validate(value) + except (OSError, TypeError, ValueError, yaml.YAMLError) as exc: + raise RealFeatureIngestionError( + f"real feature ingestion profile is invalid: {type(exc).__name__}" + ) from exc + if profile.target.table_location != ( + "s3://gda-metadata-warehouse/warehouse/cultural_heritage/cultural_districts" + ): + raise RealFeatureIngestionError("real feature target location drifted") + return profile + + +def _load_dependencies( + profile: RealFeatureIngestionProfile, +) -> tuple[dict[str, Any], m310.SparkObjectStoreInteroperabilityProfile]: + m321_profile = _resolve_repo_path(profile.dependencies.m321_profile_path) + m321_evidence_path = _resolve_repo_path(profile.dependencies.m321_evidence_path) + report = m321.build_validation_report( + profile_path=m321_profile, + evidence_path=m321_evidence_path, + ) + if report.get("status") != "valid": + raise RealFeatureIngestionError("M3-21 dependency evidence is invalid") + evidence = _load_json_object(m321_evidence_path) + if evidence.get("evidence_sha256") != M321_EVIDENCE_SHA256: + raise RealFeatureIngestionError("M3-21 dependency evidence SHA drifted") + if evidence.get("source_feature_rows_ingested") is not False: + raise RealFeatureIngestionError("M3-21 predecessor is not an empty-table promotion") + + m310_profile_path = _resolve_repo_path(profile.dependencies.m310_profile_path) + m310_evidence_path = _resolve_repo_path(profile.dependencies.m310_evidence_path) + object_report = m310.build_validation_report( + profile_path=m310_profile_path, + evidence_path=m310_evidence_path, + ) + if object_report.get("errors"): + raise RealFeatureIngestionError("M3-10 dependency evidence is invalid") + object_evidence = _load_json_object(m310_evidence_path) + if object_evidence.get("evidence_fingerprint") != M310_EVIDENCE_FINGERPRINT: + raise RealFeatureIngestionError("M3-10 dependency evidence fingerprint drifted") + return evidence, m310.load_profile(m310_profile_path) + + +def _manifest_errors(path: Path = DEFAULT_JOB_PATH) -> list[str]: + errors: list[str] = [] + try: + documents = [item for item in yaml.safe_load_all(path.read_text()) if item] + except (OSError, yaml.YAMLError) as exc: + return [f"real feature Spark manifest is invalid: {type(exc).__name__}"] + if any(_mapping(item).get("kind") == "Secret" for item in documents): + errors.append("real feature Spark manifest may not commit Secret values") + configmap = next( + (item for item in documents if _mapping(item).get("kind") == "ConfigMap"), + None, + ) + job = next((item for item in documents if _mapping(item).get("kind") == "Job"), None) + if configmap is None or job is None or len(documents) != 2: + return errors + ["real feature Spark manifest is incomplete"] + probe = str(_mapping(configmap.get("data")).get("probe.py") or "") + for marker in ( + "SedonaContext.create", + "ST_GeomFromWKB", + "ST_IsValid", + "ST_SRID", + "ST_Area", + "ST_XMin", + ".writeTo(TABLE).append()", + "existing Iceberg table is partial or content-drifted", + "GDA_REAL_FEATURE_INGESTION_RESULT=", + ): + if marker not in probe: + errors.append(f"real feature Spark probe is missing marker: {marker}") + spec = _mapping(job.get("spec")) + pod_spec = _mapping(_mapping(spec.get("template")).get("spec")) + containers = _list(pod_spec.get("containers")) + container = _mapping(containers[0]) if len(containers) == 1 else {} + volumes = _list(pod_spec.get("volumes")) + volume_names = {str(_mapping(item).get("name")) for item in volumes} + mounts = _list(container.get("volumeMounts")) + if spec.get("suspend") is not True or spec.get("backoffLimit") != 0: + errors.append("real feature Spark Job must start suspended without retries") + if pod_spec.get("automountServiceAccountToken") is not False: + errors.append("real feature Spark Job must disable token automount") + if any("persistentVolumeClaim" in _mapping(item) for item in volumes): + errors.append("real feature Spark Job may not mount a warehouse PVC") + if volume_names != {"probe", "input", "tmp"} or { + str(_mapping(item).get("name")) for item in mounts + } != volume_names: + errors.append("real feature Spark Job input boundary is incomplete") + if container.get("image") != "gisdataagent/mmfe-spark-runtime:local": + errors.append("real feature Spark image does not match the certified local runtime") + return errors + + +def build_contract_report( + *, + profile_path: Path = DEFAULT_PROFILE_PATH, + wrapper_path: Path = DEFAULT_WRAPPER_PATH, + job_path: Path = DEFAULT_JOB_PATH, +) -> dict[str, Any]: + errors: list[str] = [] + profile: RealFeatureIngestionProfile | None = None + predecessor: str | None = None + try: + profile = load_profile(profile_path) + evidence, _ = _load_dependencies(profile) + predecessor = str(evidence.get("promotion_candidate_sha256")) + except RealFeatureIngestionError as exc: + errors.append(f"real feature dependency contract is invalid: {type(exc).__name__}") + errors.extend(_manifest_errors(job_path)) + try: + wrapper = wrapper_path.read_text(encoding="utf-8") + for marker in ( + "set -euo pipefail", + "metadata_fabric_real_feature_ingestion", + '"$@"', + ): + if marker not in wrapper: + errors.append(f"real feature wrapper is missing marker: {marker}") + except OSError as exc: + errors.append(f"real feature wrapper is invalid: {type(exc).__name__}") + files = { + "implementation": _file_record(Path(__file__)), + "profile": _file_record(profile_path), + "spark_job": _file_record(job_path), + "wrapper": _file_record(wrapper_path), + } + stable = { + "schema": CONTRACT_SCHEMA, + "m321_evidence_sha256": M321_EVIDENCE_SHA256, + "m310_evidence_fingerprint": M310_EVIDENCE_FINGERPRINT, + "predecessor_promotion_candidate_sha256": predecessor, + "source_resource_version_id": str(SOURCE_RESOURCE_VERSION_ID), + "output_resource_version_id": str(OUTPUT_RESOURCE_VERSION_ID), + "authorization_action": ACTION, + "expected_feature_count": profile.source.expected_feature_count if profile else None, + "expected_srid": profile.source.expected_srid if profile else None, + "target_identity": profile.target.identity if profile else None, + "target_location": profile.target.table_location if profile else None, + "table_columns": list(GRAVITINO_COLUMNS), + "files": files, + "ingestion_persisted_to_gda_control": False, + "production_ingestion_verified": False, + "production_ready": False, + } + return { + **stable, + "contract_sha256": canonical_json_fingerprint(stable), + "local_static_contract_verified": not errors, + "status": "valid" if not errors else "invalid", + "errors": errors, + } + + +def _output_content_sha256( + *, + source_content_sha256: str, + row_set_sha256: str, +) -> str: + return canonical_json_fingerprint( + { + "schema": ROW_SET_SCHEMA, + "source_resource_version_id": str(SOURCE_RESOURCE_VERSION_ID), + "source_content_sha256": source_content_sha256, + "row_set_sha256": row_set_sha256, + "table_fields": list(ICEBERG_FIELDS), + } + ) + + +def build_source_input( + profile: RealFeatureIngestionProfile, + predecessor: Mapping[str, Any], + *, + shapefile_path: Path, + ogrinfo_path: Path, + proj_data_path: Path | None, +) -> dict[str, Any]: + inventory = build_shapefile_bundle_inventory( + shapefile_path, + source_label=profile.source.source_label, + ogrinfo_path=ogrinfo_path, + proj_data_path=proj_data_path, + ) + expected_inventory = _mapping(predecessor.get("dataset_bundle")) + if inventory != dict(expected_inventory): + raise RealFeatureIngestionError("source bundle does not match M3-21 ResourceVersion") + frame = gpd.read_file(shapefile_path) + columns = {str(item).lower(): str(item) for item in frame.columns} + identifier = columns.get(profile.source.identifier_field.lower()) + if identifier is None: + raise RealFeatureIngestionError("source identifier field is unavailable") + epsg = frame.crs.to_epsg() if frame.crs is not None else None + geometry_types = set(frame.geometry.geom_type.tolist()) + if ( + len(frame) != profile.source.expected_feature_count + or frame[identifier].isna().any() + or frame[identifier].nunique() != len(frame) + or epsg != profile.source.expected_srid + or not geometry_types.issubset({"Polygon", "MultiPolygon"}) + or not bool(frame.geometry.is_valid.all()) + or bool(frame.geometry.is_empty.any()) + or not bool(frame.geometry.has_z.all()) + ): + raise RealFeatureIngestionError("real feature source quality boundary does not match") + rows: list[dict[str, Any]] = [] + row_hashes: list[str] = [] + for _, item in frame.sort_values(identifier).iterrows(): + min_x, min_y, max_x, max_y = item.geometry.bounds + stable = { + "BSM": str(item[identifier]), + "geometry_wkb_hex": item.geometry.wkb_hex.lower(), + "srid": profile.source.expected_srid, + "min_x": min_x, + "min_y": min_y, + "max_x": max_x, + "max_y": max_y, + } + row_sha256 = canonical_json_fingerprint(stable) + rows.append({**stable, "row_sha256": row_sha256}) + row_hashes.append(row_sha256) + row_set_sha256 = canonical_json_fingerprint( + [{key: value for key, value in row.items() if key != "row_sha256"} for row in rows] + ) + source_content_sha256 = str(expected_inventory.get("content_sha256")) + output_content_sha256 = _output_content_sha256( + source_content_sha256=source_content_sha256, + row_set_sha256=row_set_sha256, + ) + raw_payload = { + "schema": ROW_SET_SCHEMA, + "source_resource_version_id": str(SOURCE_RESOURCE_VERSION_ID), + "source_content_sha256": source_content_sha256, + "output_resource_version_id": str(OUTPUT_RESOURCE_VERSION_ID), + "output_content_sha256": output_content_sha256, + "row_set_sha256": row_set_sha256, + "expected_feature_count": profile.source.expected_feature_count, + "expected_row_sha256": sorted(row_hashes), + "rows": rows, + } + payload_bytes = canonical_json_bytes(raw_payload) + if len(payload_bytes) >= 900_000: + raise RealFeatureIngestionError("real feature input exceeds the bounded ConfigMap size") + return { + "inventory": inventory, + "payload": raw_payload, + "projection": { + "schema": ROW_SET_SCHEMA, + "feature_count": len(rows), + "unique_identifier_count": len(set(row["BSM"] for row in rows)), + "valid_geometry_count": int(frame.geometry.is_valid.sum()), + "non_empty_geometry_count": int((~frame.geometry.is_empty).sum()), + "geometry_z_count": int(frame.geometry.has_z.sum()), + "geometry_types": sorted(geometry_types), + "srid": epsg, + "bounds": frame.total_bounds.tolist(), + "row_set_sha256": row_set_sha256, + "row_sha256": sorted(row_hashes), + "payload_sha256": hashlib.sha256(payload_bytes).hexdigest(), + "payload_size_bytes": len(payload_bytes), + "source_payload_recorded": False, + }, + } + + +class RealFeatureIngestionPlan(_FrozenModel): + plan_schema: Literal[PLAN_SCHEMA] = Field(default=PLAN_SCHEMA, alias="schema") + tenant_id: Literal[TENANT] + run_id: UUID + definition_version_id: UUID + source_resource_urn: str + source_resource_version_id: UUID + source_content_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + output_resource_urn: Literal[OUTPUT_RESOURCE_URN] + output_resource_version_id: UUID + output_content_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + row_set_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + expected_feature_count: Literal[20] + predecessor_promotion_candidate_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + target: m321.ObjectStoreTarget + runtime_binding: dict[str, Any] + runtime_binding_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + table_columns: tuple[dict[str, Any], ...] + writes_to_gda_control: Literal[False] = False + writes_to_legacy: Literal[False] = False + ingestion_plan_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + + @model_validator(mode="after") + def _fingerprints_match(self) -> RealFeatureIngestionPlan: + if ( + self.run_id != RUN_ID + or self.definition_version_id != DEFINITION_VERSION_ID + or self.source_resource_version_id != SOURCE_RESOURCE_VERSION_ID + or self.output_resource_version_id != OUTPUT_RESOURCE_VERSION_ID + ): + raise ValueError("real feature plan identity does not match") + if self.runtime_binding_sha256 != canonical_json_fingerprint(self.runtime_binding): + raise ValueError("runtime binding fingerprint does not match") + expected_output = _output_content_sha256( + source_content_sha256=self.source_content_sha256, + row_set_sha256=self.row_set_sha256, + ) + if self.output_content_sha256 != expected_output: + raise ValueError("output content fingerprint does not match") + if self.table_columns != GRAVITINO_COLUMNS: + raise ValueError("real feature table schema does not match") + stable = self.model_dump(mode="json", by_alias=True, exclude={"ingestion_plan_sha256"}) + if self.ingestion_plan_sha256 != canonical_json_fingerprint(stable): + raise ValueError("ingestion plan fingerprint does not match") + return self + + +def build_ingestion_plan( + profile: RealFeatureIngestionProfile, + predecessor: Mapping[str, Any], + source: Mapping[str, Any], + runtime_binding: Mapping[str, Any], +) -> RealFeatureIngestionPlan: + dataset = _mapping(predecessor.get("dataset_bundle")) + source_urn = str( + _mapping(_mapping(predecessor.get("observation")).get("plan")).get( + "resource_urn" + ) + ) + values: dict[str, Any] = { + "tenant_id": TENANT, + "run_id": RUN_ID, + "definition_version_id": DEFINITION_VERSION_ID, + "source_resource_urn": source_urn, + "source_resource_version_id": SOURCE_RESOURCE_VERSION_ID, + "source_content_sha256": dataset.get("content_sha256"), + "output_resource_urn": OUTPUT_RESOURCE_URN, + "output_resource_version_id": OUTPUT_RESOURCE_VERSION_ID, + "output_content_sha256": _mapping(source.get("payload")).get( + "output_content_sha256" + ), + "row_set_sha256": _mapping(source.get("projection")).get("row_set_sha256"), + "expected_feature_count": profile.source.expected_feature_count, + "predecessor_promotion_candidate_sha256": predecessor.get( + "promotion_candidate_sha256" + ), + "target": profile.target, + "runtime_binding": dict(runtime_binding), + "runtime_binding_sha256": canonical_json_fingerprint(runtime_binding), + "table_columns": GRAVITINO_COLUMNS, + } + stable = { + "schema": PLAN_SCHEMA, + **{ + key: value.model_dump(mode="json", by_alias=True) + if isinstance(value, BaseModel) + else str(value) + if isinstance(value, UUID) + else value + for key, value in values.items() + }, + "writes_to_gda_control": False, + "writes_to_legacy": False, + } + return RealFeatureIngestionPlan( + **values, + ingestion_plan_sha256=canonical_json_fingerprint(stable), + ) + + +def _execution_plan_artifact( + plan: RealFeatureIngestionPlan, + *, + created_at: datetime, +) -> Artifact: + manifest = { + "schema": "gda.real_feature_ingestion_execution_plan.v1", + "plan": plan.model_dump(mode="json", by_alias=True), + } + artifact_id = uuid5(RUN_ID, f"real-feature-ingestion:{plan.ingestion_plan_sha256}") + content = canonical_json_bytes(manifest) + return Artifact( + tenant_id=TENANT, + artifact_id=artifact_id, + artifact_key=f"real-feature-ingestion:{artifact_id}", + artifact_role=ArtifactRole.EXECUTION_PLAN, + storage_uri=f"postgresql://gda-control/execution-plans/{TENANT}/{artifact_id}", + media_type="application/vnd.gda.real-feature-ingestion-plan+json", + content_sha256=canonical_json_fingerprint(manifest), + size_bytes=len(content), + resource_version_id=DEFINITION_VERSION_ID, + manifest=manifest, + created_by=WORKLOAD, + created_at=created_at, + ) + + +def build_ingestion_authorization( + plan: RealFeatureIngestionPlan, + profile: RealFeatureIngestionProfile, + *, + authorized_at: datetime, +) -> tuple[PlatformRun, Artifact, Artifact, Artifact, str]: + subject = SubjectContext( + tenant_id=TENANT, + subject_id=WORKLOAD.removeprefix("workload:"), + subject_type="workload", + roles=("spatial_ingestion_executor",), + purpose="ingest one content-bound real feature slice into local Iceberg", + ) + execution_plan = _execution_plan_artifact( + plan, + created_at=authorized_at - timedelta(seconds=3), + ) + decision = PolicyDecision( + tenant_id=TENANT, + run_id=RUN_ID, + subject_context=subject, + action=ACTION, + definition_version_id=DEFINITION_VERSION_ID, + resource_version_ids=( + DEFINITION_VERSION_ID, + SOURCE_RESOURCE_VERSION_ID, + ), + execution_plan_artifact_id=execution_plan.artifact_id, + effect="allow", + policy_version_ref=profile.authorization.policy_version_ref, + evaluator_subject=profile.authorization.evaluator_subject, + requires_approval=True, + decided_at=authorized_at - timedelta(seconds=3), + expires_at=authorized_at + timedelta(days=365), + ) + policy_artifact = build_policy_decision_artifact(decision) + approval = ApprovalRecord( + tenant_id=TENANT, + run_id=RUN_ID, + definition_version_id=DEFINITION_VERSION_ID, + policy_decision_artifact_id=policy_artifact.artifact_id, + policy_decision_sha256=policy_artifact.content_sha256, + verdict="approved", + approver_subject=profile.authorization.approver_subject, + reason=profile.authorization.approval_reason, + decided_at=authorized_at - timedelta(seconds=2), + expires_at=authorized_at + timedelta(days=180), + ) + approval_artifact = build_approval_artifact(approval) + run = PlatformRun( + tenant_id=TENANT, + run_id=RUN_ID, + definition_version_id=DEFINITION_VERSION_ID, + orchestration_class="dataops", + subject_context=subject, + input_bindings=( + { + "binding_name": "source_dataset", + "resource_version_id": SOURCE_RESOURCE_VERSION_ID, + "semantic_type": "gis.cultural_districts", + }, + ), + idempotency_key=f"real-feature-ingestion:{plan.output_content_sha256}", + policy_refs=RunPolicyReferences( + policy_decision_artifact_id=policy_artifact.artifact_id, + approval_artifact_id=approval_artifact.artifact_id, + ), + submitted_at=authorized_at - timedelta(seconds=1), + ) + validate_run_authorization_evidence( + run, + policy_artifact, + approval_artifact, + execution_plan, + at=authorized_at, + expected_action=ACTION, + ) + stable = { + "run": run.model_dump(mode="json"), + "execution_plan": execution_plan.model_dump(mode="json"), + "policy_decision": policy_artifact.model_dump(mode="json"), + "approval": approval_artifact.model_dump(mode="json"), + } + return ( + run, + execution_plan, + policy_artifact, + approval_artifact, + canonical_json_fingerprint(stable), + ) + + +def validate_ingestion_authorization( + plan: RealFeatureIngestionPlan, + authorization: tuple[PlatformRun, Artifact, Artifact, Artifact, str], + *, + at: datetime, +) -> None: + run, execution_plan, policy_artifact, approval_artifact, fingerprint = authorization + if _mapping(execution_plan.manifest.get("plan")) != plan.model_dump( + mode="json", by_alias=True + ): + raise RealFeatureIngestionError("authorization does not bind the exact plan") + validate_run_authorization_evidence( + run, + policy_artifact, + approval_artifact, + execution_plan, + at=at, + expected_action=ACTION, + ) + stable = { + "run": run.model_dump(mode="json"), + "execution_plan": execution_plan.model_dump(mode="json"), + "policy_decision": policy_artifact.model_dump(mode="json"), + "approval": approval_artifact.model_dump(mode="json"), + } + if fingerprint != canonical_json_fingerprint(stable): + raise RealFeatureIngestionError("authorization fingerprint does not match") + + +def create_target_table( + rehearsal: m321.ObjectStoreProjectionRehearsal, + profile: RealFeatureIngestionProfile, + plan: RealFeatureIngestionPlan, +) -> dict[str, Any]: + if rehearsal.bounded is None: + raise RealFeatureIngestionError("bounded Gravitino identity is unavailable") + rehearsal.bounded.request( + "POST", + f"{rehearsal._schema_path(profile.target)}/tables", + json_body={ + "name": profile.target.table, + "comment": "Authorized real Chongqing cultural district Iceberg slice", + "columns": [ + { + **column, + "comment": "Content-bound cross-engine spatial field", + } + for column in GRAVITINO_COLUMNS + ], + "properties": { + "gda.resource_urn": plan.output_resource_urn, + "gda.resource_version_id": str(plan.output_resource_version_id), + "gda.content_sha256": plan.output_content_sha256, + "gda.source_resource_urn": plan.source_resource_urn, + "gda.source_resource_version_id": str(plan.source_resource_version_id), + "gda.source_content_sha256": plan.source_content_sha256, + "gda.row_set_sha256": plan.row_set_sha256, + "gda.provider_revision": "m3-22-real-feature-ingestion-v1", + }, + }, + label="bounded real feature table create", + ) + _, payload = rehearsal.bounded.request( + "GET", + rehearsal._table_path(profile.target), + label="real feature table exact readback", + ) + assert payload is not None + projection = durable._table_projection(payload) + expected = { + "name": profile.target.table, + "columns": [dict(item) for item in GRAVITINO_COLUMNS], + "resource_urn": plan.output_resource_urn, + "resource_version_id": str(plan.output_resource_version_id), + "content_sha256": plan.output_content_sha256, + "provider_revision": "m3-22-real-feature-ingestion-v1", + } + if projection != expected: + raise RealFeatureIngestionError("real feature table projection drifted") + properties = _mapping(_mapping(payload.get("table")).get("properties")) + drifted_source_properties: list[str] = [] + for key, expected_value in ( + ("gda.source_resource_urn", plan.source_resource_urn), + ("gda.source_resource_version_id", str(plan.source_resource_version_id)), + ("gda.source_content_sha256", plan.source_content_sha256), + ("gda.row_set_sha256", plan.row_set_sha256), + ): + if properties.get(key) != expected_value: + drifted_source_properties.append(key) + if drifted_source_properties: + raise RealFeatureIngestionError( + "real feature source binding drifted: " + + ", ".join(drifted_source_properties) + ) + return { + "status": "created", + "mutation_count": 1, + "mutations": ["gravitino.table.create"], + "table_projection": projection, + "table_projection_sha256": canonical_json_fingerprint(projection), + "source_binding_verified": True, + } + + +def _run_spark_ingestion( + runtime: m310.IsolatedSparkObjectStoreRuntime, + *, + input_payload: Mapping[str, Any], +) -> dict[str, Any]: + namespace = runtime.profile.cluster.rehearsal_namespace + input_resource = { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": "real-feature-ingestion-input", + "namespace": namespace, + }, + "immutable": True, + "data": { + "ingestion.json": json.dumps( + input_payload, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + }, + } + runtime.kubectl.run( + ["create", "-f", "-"], + input_text=json.dumps(input_resource, ensure_ascii=True, separators=(",", ":")), + label="ephemeral real feature input create", + ) + runtime.kubectl.run( + ["apply", "-f", str(DEFAULT_JOB_PATH)], + label="real feature Spark Job apply", + ) + job_name = "real-feature-ingestion-probe" + runtime.kubectl.run( + [ + "-n", + namespace, + "patch", + "job", + job_name, + "--type=merge", + "-p", + '{"spec":{"suspend":false}}', + ], + label="real feature Spark Job release", + ) + deadline = time.monotonic() + 900 + terminal: str | None = None + while time.monotonic() < deadline: + current = runtime.kubectl.get_json( + ["-n", namespace, "get", "job", job_name], + label="real feature Spark Job wait", + ) + assert current is not None + for condition in _list(_mapping(current.get("status")).get("conditions")): + item = _mapping(condition) + if item.get("status") == "True" and item.get("type") in {"Complete", "Failed"}: + terminal = str(item.get("type")) + break + if terminal is not None: + break + time.sleep(2) + job = runtime.kubectl.get_json( + ["-n", namespace, "get", "job", job_name], + label="real feature Spark Job observation", + ) + pod_list = runtime.kubectl.get_json( + ["-n", namespace, "get", "pods", "-l", f"job-name={job_name}"], + label="real feature Spark pod observation", + ) + assert job is not None and pod_list is not None + pod = m310._single_list_item(pod_list, "real feature Spark Job") + pod_name = str(_mapping(pod.get("metadata")).get("name")) + logs = runtime.kubectl.run( + ["-n", namespace, "logs", pod_name, "-c", "spark"], + expected=frozenset({0, 1}), + timeout=120, + label="real feature Spark result collection", + ) + lines = [ + line.removeprefix("GDA_REAL_FEATURE_INGESTION_RESULT=") + for line in logs.stdout.splitlines() + if line.startswith("GDA_REAL_FEATURE_INGESTION_RESULT=") + ] + result: dict[str, Any] | None = None + if len(lines) == 1: + candidate = json.loads(lines[0]) + if isinstance(candidate, dict): + result = candidate + diagnostic: list[str] = [] + if result is None: + for line in logs.stdout.splitlines()[-100:]: + if any( + marker in line.lower() + for marker in ( + "access-key", + "authorization:", + "credential", + "password", + "secret", + "token", + ) + ): + diagnostic.append("") + else: + diagnostic.append(line[:1000]) + pod_spec = _mapping(pod.get("spec")) + claims = sorted( + str(_mapping(_mapping(item).get("persistentVolumeClaim")).get("claimName")) + for item in _list(pod_spec.get("volumes")) + if _mapping(_mapping(item).get("persistentVolumeClaim")).get("claimName") + ) + container = m310._container_status(pod, "spark") + status = _mapping(job.get("status")) + return { + "wait_completed": terminal == "Complete", + "terminal_condition": terminal, + "job": { + "name": job_name, + "uid": _mapping(job.get("metadata")).get("uid"), + "succeeded": status.get("succeeded", 0), + "failed": status.get("failed", 0), + "completion_time": status.get("completionTime"), + }, + "pod": { + "name": pod_name, + "uid": _mapping(pod.get("metadata")).get("uid"), + "phase": _mapping(pod.get("status")).get("phase"), + "node_name": pod_spec.get("nodeName"), + "service_account": pod_spec.get("serviceAccountName"), + "service_account_automount_disabled": ( + pod_spec.get("automountServiceAccountToken") is False + ), + "image": container.get("image"), + "image_id": container.get("imageID"), + "persistent_volume_claims": claims, + }, + "result_line_count": len(lines), + "log_sha256": hashlib.sha256(logs.stdout.encode()).hexdigest(), + "log_recorded": False, + "failure_diagnostic": diagnostic, + "result": result, + } + + +def observe_ingested_table( + runtime: m310.IsolatedSparkObjectStoreRuntime, + profile: RealFeatureIngestionProfile, + *, + endpoint_url: str, + object_store_user: SecretStr, + object_store_material: SecretStr, +) -> dict[str, Any]: + client = runtime._s3_client( + endpoint_url=endpoint_url, + object_store_user=object_store_user, + object_store_material=object_store_material, + ) + try: + objects: list[dict[str, Any]] = [] + continuation: str | None = None + while True: + request: dict[str, Any] = { + "Bucket": profile.target.bucket, + "Prefix": profile.target.object_prefix, + } + if continuation: + request["ContinuationToken"] = continuation + response = client.list_objects_v2(**request) + for item in response.get("Contents") or []: + objects.append( + { + "key": item.get("Key"), + "size": item.get("Size"), + "etag": str(item.get("ETag") or "").strip('"'), + } + ) + if response.get("IsTruncated") is not True: + break + continuation = response.get("NextContinuationToken") + if not continuation: + raise RealFeatureIngestionError("S3 listing continuation is invalid") + objects.sort(key=lambda item: str(item.get("key"))) + data_keys = sorted( + str(item["key"]) + for item in objects + if str(item.get("key") or "").endswith(".parquet") + ) + metadata_keys = sorted( + str(item["key"]) + for item in objects + if str(item.get("key") or "").endswith(".metadata.json") + ) + manifest_keys = sorted( + str(item["key"]) + for item in objects + if str(item.get("key") or "").endswith(".avro") + ) + if len(data_keys) != 1 or not metadata_keys or not manifest_keys: + raise RealFeatureIngestionError("direct S3 Iceberg object classes are incomplete") + latest_key = metadata_keys[-1] + response = client.get_object(Bucket=profile.target.bucket, Key=latest_key) + body = response["Body"].read() + metadata = json.loads(body) + if not isinstance(metadata, dict): + raise TypeError("Iceberg metadata must be an object") + current_schema_id = metadata.get("current-schema-id") + current_schema = next( + ( + _mapping(item) + for item in _list(metadata.get("schemas")) + if _mapping(item).get("schema-id") == current_schema_id + ), + {}, + ) + fields = tuple( + { + "name": _mapping(item).get("name"), + "required": _mapping(item).get("required"), + "type": _mapping(item).get("type"), + } + for item in _list(current_schema.get("fields")) + ) + if ( + fields != ICEBERG_FIELDS + or metadata.get("location") != profile.target.table_location + or metadata.get("current-snapshot-id") is None + ): + raise RealFeatureIngestionError("direct S3 Iceberg metadata drifted") + return { + "bucket": profile.target.bucket, + "prefix": profile.target.object_prefix, + "object_count": len(objects), + "objects": objects, + "object_inventory_sha256": canonical_json_fingerprint(objects), + "data_keys": data_keys, + "metadata_keys": metadata_keys, + "manifest_keys": manifest_keys, + "latest_metadata": { + "key": latest_key, + "body_sha256": hashlib.sha256(body).hexdigest(), + "size_bytes": len(body), + "location": metadata.get("location"), + "current_snapshot_id": metadata.get("current-snapshot-id"), + "current_schema_id": current_schema_id, + "fields": list(fields), + }, + "source_feature_payload_recorded": False, + "material_recorded": False, + } + finally: + client.close() + + +def build_output_contracts( + plan: RealFeatureIngestionPlan, + spark: Mapping[str, Any], + store: Mapping[str, Any], + *, + created_at: datetime, +) -> dict[str, Any]: + result = _mapping(spark.get("result")) + quality_metrics = dict(_mapping(result.get("quality"))) + first = _mapping(result.get("first_execution")) + snapshots = _list(first.get("snapshots")) + snapshot_id = _mapping(snapshots[0]).get("snapshot_id") if snapshots else None + output = ResourceVersion( + tenant_id=TENANT, + resource_urn=OUTPUT_RESOURCE_URN, + resource_version_id=OUTPUT_RESOURCE_VERSION_ID, + version_key=f"rows-{plan.output_content_sha256[:12]}", + content_sha256=plan.output_content_sha256, + authority_version_ref={ + "catalog": plan.target.catalog, + "schema": plan.target.schema_name, + "table": plan.target.table, + "snapshot_id": snapshot_id, + "row_set_sha256": plan.row_set_sha256, + }, + created_by=WORKLOAD, + created_at=created_at, + ) + data_objects = [ + item + for item in _list(store.get("objects")) + if str(_mapping(item).get("key") or "").endswith(".parquet") + ] + output_artifact_id = uuid5(RUN_ID, f"output:{plan.output_content_sha256}") + output_artifact = Artifact( + tenant_id=TENANT, + artifact_id=output_artifact_id, + artifact_key=f"cultural-districts-iceberg:{output_artifact_id}", + artifact_role=ArtifactRole.OUTPUT, + storage_uri=plan.target.table_location, + media_type="application/vnd.apache.iceberg.table", + content_sha256=plan.output_content_sha256, + size_bytes=sum(int(_mapping(item).get("size") or 0) for item in data_objects), + run_id=RUN_ID, + resource_version_id=OUTPUT_RESOURCE_VERSION_ID, + manifest={ + "snapshot_id": snapshot_id, + "row_set_sha256": plan.row_set_sha256, + "feature_count": plan.expected_feature_count, + "data_file_count": len(data_objects), + }, + created_by=WORKLOAD, + created_at=created_at, + ) + latest = _mapping(store.get("latest_metadata")) + quality_artifact_id = uuid5(RUN_ID, f"quality-evidence:{latest.get('body_sha256')}") + quality_artifact = Artifact( + tenant_id=TENANT, + artifact_id=quality_artifact_id, + artifact_key=f"real-feature-quality:{quality_artifact_id}", + artifact_role=ArtifactRole.EVIDENCE, + storage_uri=f"s3://{plan.target.bucket}/{latest.get('key')}", + media_type="application/vnd.apache.iceberg+json", + content_sha256=str(latest.get("body_sha256")), + size_bytes=int(latest.get("size_bytes") or 0), + run_id=RUN_ID, + resource_version_id=OUTPUT_RESOURCE_VERSION_ID, + manifest={ + "rule_version_ref": "quality://gda/spatial/real-feature-ingestion/v1", + "metrics": quality_metrics, + "row_set_sha256": plan.row_set_sha256, + }, + created_by=WORKLOAD, + created_at=created_at, + ) + evaluated_at = created_at + timedelta(seconds=1) + quality_sha = quality_result_fingerprint( + tenant_id=TENANT, + run_id=RUN_ID, + resource_version_id=OUTPUT_RESOURCE_VERSION_ID, + rule_version_ref="quality://gda/spatial/real-feature-ingestion/v1", + verdict=QualityVerdict.PASSED, + metrics=quality_metrics, + evidence_artifact_id=quality_artifact_id, + evaluated_by=QUALITY_EVALUATOR, + evaluated_at=evaluated_at, + ) + quality = QualityResult( + tenant_id=TENANT, + quality_result_id=uuid5(RUN_ID, f"quality:{quality_sha}"), + run_id=RUN_ID, + resource_version_id=OUTPUT_RESOURCE_VERSION_ID, + rule_version_ref="quality://gda/spatial/real-feature-ingestion/v1", + verdict=QualityVerdict.PASSED, + metrics=quality_metrics, + evidence_artifact_id=quality_artifact_id, + result_sha256=quality_sha, + evaluated_by=QUALITY_EVALUATOR, + evaluated_at=evaluated_at, + ) + lineage_values = { + "event_type": LineageEventType.DERIVE.value, + "source_resource_version_id": str(SOURCE_RESOURCE_VERSION_ID), + "target_resource_version_id": str(OUTPUT_RESOURCE_VERSION_ID), + "run_id": str(RUN_ID), + "definition_version_id": str(DEFINITION_VERSION_ID), + "artifact_id": str(output_artifact_id), + "producer": WORKLOAD, + "facets": { + "row_set_sha256": plan.row_set_sha256, + "snapshot_id": snapshot_id, + "feature_count": plan.expected_feature_count, + }, + "occurred_at": created_at.isoformat().replace("+00:00", "Z"), + } + lineage_sha = canonical_json_fingerprint(lineage_values) + lineage = LineageEvent( + tenant_id=TENANT, + lineage_event_id=uuid5(RUN_ID, f"lineage:{lineage_sha}"), + event_type=LineageEventType.DERIVE, + source_resource_version_id=SOURCE_RESOURCE_VERSION_ID, + target_resource_version_id=OUTPUT_RESOURCE_VERSION_ID, + producer=WORKLOAD, + event_sha256=lineage_sha, + run_id=RUN_ID, + definition_version_id=DEFINITION_VERSION_ID, + artifact_id=output_artifact_id, + facets=lineage_values["facets"], + occurred_at=created_at, + ) + return { + "output_resource_version": output.model_dump(mode="json"), + "output_artifact": output_artifact.model_dump(mode="json"), + "quality_evidence_artifact": quality_artifact.model_dump(mode="json"), + "quality_result": quality.model_dump(mode="json"), + "lineage_event": lineage.model_dump(mode="json"), + "persisted_to_gda_control": False, + } + + +def _spark_errors( + spark: Mapping[str, Any], + plan: RealFeatureIngestionPlan, + source: Mapping[str, Any], + *, + expected_authorization_sha256: str, +) -> list[str]: + errors: list[str] = [] + pod = _mapping(spark.get("pod")) + result = _mapping(spark.get("result")) + first = _mapping(result.get("first_execution")) + replay_result = _mapping(result.get("immediate_replay")) + expected_hashes = _list(_mapping(source.get("projection")).get("row_sha256")) + quality = _mapping(result.get("quality")) + if ( + spark.get("wait_completed") is not True + or spark.get("terminal_condition") != "Complete" + or _mapping(spark.get("job")).get("succeeded") != 1 + or _mapping(spark.get("job")).get("failed") != 0 + or spark.get("result_line_count") != 1 + or spark.get("failure_diagnostic") != [] + ): + errors.append("real feature Spark Job did not complete exactly once") + if ( + pod.get("node_name") != "desktop-worker" + or pod.get("service_account") != "spark-object-store-probe" + or pod.get("service_account_automount_disabled") is not True + or pod.get("persistent_volume_claims") != [] + ): + errors.append("real feature Spark execution boundary does not match") + if ( + result.get("schema") != PROBE_RESULT_SCHEMA + or result.get("plan_sha256") != plan.ingestion_plan_sha256 + or result.get("source_resource_version_id") != str(SOURCE_RESOURCE_VERSION_ID) + or result.get("source_content_sha256") != plan.source_content_sha256 + or result.get("output_resource_version_id") != str(OUTPUT_RESOURCE_VERSION_ID) + or result.get("output_content_sha256") != plan.output_content_sha256 + or result.get("row_set_sha256") != plan.row_set_sha256 + or result.get("spark_version") != "3.5.0" + or result.get("sedona_version") != "1.9.0" + or result.get("iceberg_runtime") != "1.6.1" + or tuple(result.get("table_columns") or ()) != SPARK_COLUMNS + or result.get("source_payload_recorded") is not False + or result.get("material_recorded") is not False + ): + errors.append("real feature Spark result binding does not match") + if any( + quality.get(key) != plan.expected_feature_count + for key in ( + "feature_count", + "unique_bsm_count", + "valid_geometry_count", + "srid_match_count", + "positive_area_count", + "bbox_match_count", + ) + ): + errors.append("Sedona spatial quality gate did not verify every feature") + if ( + first.get("status") != "appended" + or first.get("mutation_count") != 1 + or replay_result.get("status") != "no_op" + or replay_result.get("mutation_count") != 0 + or first.get("row_sha256") != expected_hashes + or replay_result.get("row_sha256") != expected_hashes + or first.get("snapshots") != replay_result.get("snapshots") + or first.get("data_files") != replay_result.get("data_files") + or len(_list(first.get("snapshots"))) != 1 + or len(_list(first.get("data_files"))) != 1 + ): + errors.append("real feature ingestion replay is not an exact no-op") + if result.get("authorization_sha256") != expected_authorization_sha256: + errors.append("real feature Spark result is not authorization-bound") + return errors + + +def _object_store_errors( + store: Mapping[str, Any], + spark: Mapping[str, Any], + profile: RealFeatureIngestionProfile, +) -> list[str]: + errors: list[str] = [] + result = _mapping(spark.get("result")) + first = _mapping(result.get("first_execution")) + snapshots = _list(first.get("snapshots")) + data_files = _list(first.get("data_files")) + expected_data_keys = sorted( + str(_mapping(item).get("file_path") or "").removeprefix( + f"s3://{profile.target.bucket}/" + ) + for item in data_files + ) + if ( + store.get("bucket") != profile.target.bucket + or store.get("prefix") != profile.target.object_prefix + or store.get("data_keys") != expected_data_keys + or len(_list(store.get("data_keys"))) != 1 + or not store.get("metadata_keys") + or not store.get("manifest_keys") + or _mapping(store.get("latest_metadata")).get("location") + != profile.target.table_location + or tuple(_mapping(store.get("latest_metadata")).get("fields") or ()) + != ICEBERG_FIELDS + or not snapshots + or _mapping(store.get("latest_metadata")).get("current_snapshot_id") + != _mapping(snapshots[0]).get("snapshot_id") + or store.get("source_feature_payload_recorded") is not False + or store.get("material_recorded") is not False + ): + errors.append("direct S3 Iceberg data projection does not match Spark readback") + return errors + + +def build_evidence( + observation: Mapping[str, Any], + *, + profile: RealFeatureIngestionProfile, +) -> dict[str, Any]: + errors: list[str] = [] + try: + replay._reject_sensitive_fields(observation) + except ValueError: + errors.append("real feature observation contains sensitive material") + if observation.get("schema") != OBSERVATION_SCHEMA: + errors.append("real feature observation schema does not match") + contract = build_contract_report() + if _mapping(observation.get("contract")).get("contract_sha256") != contract.get( + "contract_sha256" + ): + errors.append("real feature contract binding does not match") + dataset = dict(_mapping(observation.get("dataset_bundle"))) + errors.extend(validate_shapefile_bundle_inventory(dataset)) + try: + plan = RealFeatureIngestionPlan.model_validate(observation.get("plan")) + except ValueError: + errors.append("real feature ingestion plan is invalid") + plan = None + source = _mapping(observation.get("source_projection")) + authorization = _mapping(observation.get("authorization")) + if plan is not None: + if ( + plan.source_content_sha256 != dataset.get("content_sha256") + or source.get("feature_count") != plan.expected_feature_count + or source.get("unique_identifier_count") != plan.expected_feature_count + or source.get("valid_geometry_count") != plan.expected_feature_count + or source.get("non_empty_geometry_count") != plan.expected_feature_count + or source.get("geometry_z_count") != plan.expected_feature_count + or source.get("srid") != 4490 + or source.get("row_set_sha256") != plan.row_set_sha256 + or len(_list(source.get("row_sha256"))) != plan.expected_feature_count + or source.get("source_payload_recorded") is not False + ): + errors.append("real feature source projection does not match the plan") + errors.extend( + _spark_errors( + _mapping(observation.get("spark")), + plan, + {"projection": source}, + expected_authorization_sha256=str( + authorization.get("authorization_sha256") or "" + ), + ) + ) + errors.extend( + _object_store_errors( + _mapping(observation.get("object_store")), + _mapping(observation.get("spark")), + profile, + ) + ) + if ( + authorization.get("action") != ACTION + or authorization.get("provider_apply_authorized") is not True + or not authorization.get("authorization_sha256") + ): + errors.append("real feature ingestion authorization is incomplete") + table_create = _mapping(observation.get("table_create")) + if ( + table_create.get("status") != "created" + or table_create.get("mutation_count") != 1 + or table_create.get("mutations") != ["gravitino.table.create"] + or table_create.get("source_binding_verified") is not True + ): + errors.append("real feature target table was not created exactly once") + contracts = _mapping(observation.get("output_contracts")) + try: + output = ResourceVersion.model_validate(contracts.get("output_resource_version")) + output_artifact = Artifact.model_validate(contracts.get("output_artifact")) + quality_artifact = Artifact.model_validate(contracts.get("quality_evidence_artifact")) + quality = QualityResult.model_validate(contracts.get("quality_result")) + lineage = LineageEvent.model_validate(contracts.get("lineage_event")) + if ( + plan is None + or output.resource_version_id != OUTPUT_RESOURCE_VERSION_ID + or output.content_sha256 != plan.output_content_sha256 + or output_artifact.resource_version_id != OUTPUT_RESOURCE_VERSION_ID + or output_artifact.content_sha256 != plan.output_content_sha256 + or quality.evidence_artifact_id != quality_artifact.artifact_id + or quality.resource_version_id != OUTPUT_RESOURCE_VERSION_ID + or quality.verdict != QualityVerdict.PASSED + or quality.evaluated_by == WORKLOAD + or lineage.event_type != LineageEventType.DERIVE + or lineage.source_resource_version_id != SOURCE_RESOURCE_VERSION_ID + or lineage.target_resource_version_id != OUTPUT_RESOURCE_VERSION_ID + or lineage.artifact_id != output_artifact.artifact_id + or contracts.get("persisted_to_gda_control") is not False + ): + errors.append("real feature output/quality/lineage contracts drifted") + except ValueError: + errors.append("real feature output contracts are invalid") + runtime = _mapping(observation.get("runtime_checks")) + if ( + runtime.get("all_runtime_port_forwards_stopped") is not True + or runtime.get("namespace_delete_completed") is not True + or runtime.get("namespace_absent") is not True + or runtime.get("persistent_volumes_absent") is not True + or runtime.get("provider_objects_retained") is not False + or runtime.get("object_store_objects_retained") is not False + or runtime.get("material_recorded") is not False + ): + errors.append("real feature ingestion runtime cleanup is incomplete") + verified = not errors + stable = { + "schema": EVIDENCE_SCHEMA, + "environment": "local_docker_desktop", + "status": "local_real_feature_ingestion_verified" if verified else "blocked", + "contract_sha256": contract.get("contract_sha256"), + "m321_evidence_sha256": M321_EVIDENCE_SHA256, + "m310_evidence_fingerprint": M310_EVIDENCE_FINGERPRINT, + "dataset_bundle": dataset, + "source_projection": dict(source), + "source_resource_version_id": str(SOURCE_RESOURCE_VERSION_ID), + "output_resource_version_id": str(OUTPUT_RESOURCE_VERSION_ID), + "predecessor_promotion_candidate_sha256": ( + plan.predecessor_promotion_candidate_sha256 if plan else None + ), + "local_real_feature_ingestion_verified": verified, + "real_dataset_resource_version_bound": verified, + "authorized_spark_execution_verified": verified, + "sedona_spatial_quality_verified": verified, + "iceberg_single_snapshot_verified": verified, + "exact_ingestion_replay_no_op_verified": verified, + "direct_object_store_data_verified": verified, + "path_free_lineage_candidate_verified": verified, + **{claim: False for claim in FALSE_CLAIMS}, + "observation": dict(observation), + "errors": errors, + } + return {**stable, "evidence_sha256": canonical_json_fingerprint(stable)} + + +def verify_evidence_integrity(evidence: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + try: + replay._reject_sensitive_fields(evidence) + except ValueError: + errors.append("real feature evidence contains sensitive material") + stable = {key: value for key, value in evidence.items() if key != "evidence_sha256"} + if evidence.get("evidence_sha256") != canonical_json_fingerprint(stable): + errors.append("real feature evidence SHA-256 does not match") + if evidence.get("schema") != EVIDENCE_SCHEMA or evidence.get("errors") != []: + errors.append("real feature evidence is not verified") + for claim in ( + "local_real_feature_ingestion_verified", + "real_dataset_resource_version_bound", + "authorized_spark_execution_verified", + "sedona_spatial_quality_verified", + "iceberg_single_snapshot_verified", + "exact_ingestion_replay_no_op_verified", + "direct_object_store_data_verified", + "path_free_lineage_candidate_verified", + ): + if evidence.get(claim) is not True: + errors.append(f"real feature evidence claim is false: {claim}") + for claim in FALSE_CLAIMS: + if evidence.get(claim) is not False: + errors.append(f"real feature evidence may not claim {claim}") + serialized = json.dumps(evidence, ensure_ascii=True, sort_keys=True) + for forbidden in ( + "/Users/", + "/home/", + "Downloads/", + ".tmp/", + "geometry_wkb_hex", + '"rows"', + '"password"', + '"secret"', + '"token"', + '"access_key"', + '"access-key"', + ): + if forbidden in serialized: + errors.append("real feature evidence contains source or secret material") + break + return errors + + +def run_live_rehearsal( + *, + profile_path: Path, + shapefile_path: Path, + ogrinfo_path: Path, + proj_data_path: Path | None, +) -> dict[str, Any]: + profile = load_profile(profile_path) + predecessor, runtime_profile = _load_dependencies(profile) + contract = build_contract_report(profile_path=profile_path) + if contract.get("status") != "valid": + raise RealFeatureIngestionError("real feature static contract is invalid") + source = build_source_input( + profile, + predecessor, + shapefile_path=shapefile_path, + ogrinfo_path=ogrinfo_path, + proj_data_path=proj_data_path, + ) + admin_material = SecretStr(secrets.token_urlsafe(24)) + database_material = SecretStr(secrets.token_urlsafe(24)) + user_material = SecretStr(secrets.token_urlsafe(24)) + object_store_user = SecretStr("gda" + secrets.token_hex(8)) + object_store_material = SecretStr(secrets.token_urlsafe(32)) + runtime = m310.IsolatedSparkObjectStoreRuntime(runtime_profile) + object_forward: Any = None + gravitino_forward: Any = None + rehearsal: m321.ObjectStoreProjectionRehearsal | None = None + object_forward_stopped = False + gravitino_forward_stopped = False + cleanup = { + "namespace_delete_completed": False, + "namespace_absent": False, + "persistent_volumes_absent": False, + "provider_objects_retained": True, + "object_store_objects_retained": True, + } + initial_runtime: dict[str, Any] | None = None + object_store_prepared: dict[str, Any] | None = None + plan: RealFeatureIngestionPlan | None = None + authorization: tuple[PlatformRun, Artifact, Artifact, Artifact, str] | None = None + bootstrap: dict[str, Any] | None = None + table_create: dict[str, Any] | None = None + spark: dict[str, Any] | None = None + store: dict[str, Any] | None = None + output_contracts: dict[str, Any] | None = None + runtime_binding: dict[str, Any] | None = None + cluster_uid: str | None = None + authorized_at: datetime | None = None + try: + initial_runtime = runtime.start( + admin_material=admin_material, + database_material=database_material, + object_store_user=object_store_user, + object_store_material=object_store_material, + ) + cluster = runtime.kubectl.get_json( + ["get", "namespace", "kube-system"], + label="real feature cluster identity", + ) + cluster_uid = str(_mapping(_mapping(cluster).get("metadata")).get("uid")) + runtime_binding = m321._provider_runtime_binding( + initial_runtime, + cluster_uid=cluster_uid, + target=profile.target, + ) + plan = build_ingestion_plan(profile, predecessor, source, runtime_binding) + authorized_at = datetime.now(UTC) + authorization = build_ingestion_authorization( + plan, + profile, + authorized_at=authorized_at, + ) + validate_ingestion_authorization(plan, authorization, at=authorized_at) + + object_forward = m321.provider_metrics._PortForward( + kubectl="kubectl", + context=runtime_profile.cluster.context, + namespace=runtime_profile.cluster.rehearsal_namespace, + service=runtime_profile.runtime.object_store_service, + target_port=runtime_profile.runtime.object_store_service_port, + ) + object_forward.start() + endpoint = f"http://127.0.0.1:{object_forward.local_port}" + object_store_prepared = runtime.prepare_object_store( + endpoint_url=endpoint, + object_store_user=object_store_user, + object_store_material=object_store_material, + ) + + gravitino_forward = m321.provider_metrics._PortForward( + kubectl="kubectl", + context=runtime_profile.cluster.context, + namespace=runtime_profile.cluster.rehearsal_namespace, + service=runtime_profile.runtime.service, + target_port=runtime_profile.runtime.gravitino_service_port, + ) + gravitino_forward.start() + rehearsal = m321.ObjectStoreProjectionRehearsal( + base_url=f"http://127.0.0.1:{gravitino_forward.local_port}/api", + admin_name=profile.identity.service_admin, + admin_material=admin_material, + ) + bootstrap = rehearsal.bootstrap( + profile, + database_material=database_material, + user_material=user_material, + object_store_user=object_store_user, + object_store_material=object_store_material, + ) + table_create = create_target_table(rehearsal, profile, plan) + rehearsal.close() + rehearsal = None + gravitino_forward_stopped = gravitino_forward.stop() + gravitino_forward = None + + run, execution_plan, policy, approval, auth_sha = authorization + input_payload = { + **dict(_mapping(source.get("payload"))), + "plan_sha256": plan.ingestion_plan_sha256, + "authorization_sha256": auth_sha, + } + spark = _run_spark_ingestion(runtime, input_payload=input_payload) + store = observe_ingested_table( + runtime, + profile, + endpoint_url=endpoint, + object_store_user=object_store_user, + object_store_material=object_store_material, + ) + output_contracts = build_output_contracts( + plan, + spark, + store, + created_at=datetime.now(UTC), + ) + finally: + if rehearsal is not None: + rehearsal.close() + if gravitino_forward is not None: + gravitino_forward_stopped = gravitino_forward.stop() + if object_forward is not None: + object_forward_stopped = object_forward.stop() + cleanup = runtime.cleanup() + required = ( + initial_runtime, + object_store_prepared, + plan, + authorization, + bootstrap, + table_create, + spark, + store, + output_contracts, + runtime_binding, + cluster_uid, + authorized_at, + ) + if any(item is None for item in required): + raise RealFeatureIngestionError("real feature rehearsal outcome is incomplete") + assert plan is not None and authorization is not None + run, execution_plan, policy, approval, auth_sha = authorization + observation = { + "schema": OBSERVATION_SCHEMA, + "observed_at": datetime.now(UTC).isoformat(), + "contract": { + "contract_sha256": contract["contract_sha256"], + "m321_evidence_sha256": M321_EVIDENCE_SHA256, + "m310_evidence_fingerprint": M310_EVIDENCE_FINGERPRINT, + }, + "dataset_bundle": source["inventory"], + "source_projection": source["projection"], + "cluster_uid": cluster_uid, + "runtime_binding": runtime_binding, + "runtime_binding_sha256": canonical_json_fingerprint(runtime_binding), + "initial_runtime": initial_runtime, + "object_store_prepared": object_store_prepared, + "plan": plan.model_dump(mode="json", by_alias=True), + "authorization": { + "action": ACTION, + "provider_apply_authorized": True, + "authorization_sha256": auth_sha, + "run_id": str(run.run_id), + "execution_plan_artifact_id": str(execution_plan.artifact_id), + "policy_decision_artifact_id": str(policy.artifact_id), + "approval_artifact_id": str(approval.artifact_id), + }, + "bootstrap": bootstrap, + "table_create": table_create, + "spark": spark, + "object_store": store, + "output_contracts": output_contracts, + "runtime_checks": { + **cleanup, + "all_runtime_port_forwards_stopped": ( + object_forward_stopped and gravitino_forward_stopped + ), + "material_recorded": False, + }, + } + return build_evidence(observation, profile=profile) + + +def build_validation_report( + *, + profile_path: Path = DEFAULT_PROFILE_PATH, + evidence_path: Path = DEFAULT_EVIDENCE_PATH, +) -> dict[str, Any]: + contract = build_contract_report(profile_path=profile_path) + errors = list(contract["errors"]) + evidence: dict[str, Any] | None = None + try: + evidence = _load_json_object(evidence_path) + errors.extend(verify_evidence_integrity(evidence)) + observed_contract = _mapping(_mapping(evidence.get("observation")).get("contract")) + if observed_contract.get("contract_sha256") != contract.get("contract_sha256"): + errors.append("real feature evidence contract SHA drift") + except (OSError, TypeError, ValueError, json.JSONDecodeError) as exc: + errors.append(f"real feature evidence is invalid: {type(exc).__name__}") + verified = not errors + return { + "schema": VALIDATION_SCHEMA, + "status": "valid" if verified else "invalid", + "local_static_contract_verified": contract["local_static_contract_verified"], + "local_real_feature_ingestion_verified": ( + verified + and evidence is not None + and evidence.get("local_real_feature_ingestion_verified") is True + ), + "sedona_spatial_quality_verified": ( + verified + and evidence is not None + and evidence.get("sedona_spatial_quality_verified") is True + ), + "exact_ingestion_replay_no_op_verified": ( + verified + and evidence is not None + and evidence.get("exact_ingestion_replay_no_op_verified") is True + ), + "ingestion_persisted_to_gda_control": False, + "protected_workload_identity_verified": False, + "production_object_store_verified": False, + "production_ingestion_verified": False, + "production_ready": False, + "contract_sha256": contract["contract_sha256"], + "evidence_sha256": evidence.get("evidence_sha256") if evidence else None, + "errors": errors, + } + + +def _write_json(path: Path, payload: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(payload, ensure_ascii=True, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + validate = subparsers.add_parser("validate") + validate.add_argument("--profile", type=Path, default=DEFAULT_PROFILE_PATH) + validate.add_argument("--evidence", type=Path, default=DEFAULT_EVIDENCE_PATH) + contract = subparsers.add_parser("contract") + contract.add_argument("--profile", type=Path, default=DEFAULT_PROFILE_PATH) + live = subparsers.add_parser("live") + live.add_argument("--profile", type=Path, default=DEFAULT_PROFILE_PATH) + live.add_argument("--shapefile", type=Path, required=True) + live.add_argument("--ogrinfo", type=Path, required=True) + live.add_argument("--proj-data", type=Path) + live.add_argument("--output", type=Path, default=DEFAULT_EVIDENCE_PATH) + args = parser.parse_args(argv) + try: + if args.command == "contract": + report = build_contract_report(profile_path=args.profile) + elif args.command == "live": + report = run_live_rehearsal( + profile_path=args.profile, + shapefile_path=args.shapefile, + ogrinfo_path=args.ogrinfo, + proj_data_path=args.proj_data, + ) + _write_json(args.output, report) + else: + report = build_validation_report( + profile_path=args.profile, + evidence_path=args.evidence, + ) + print(json.dumps(report, ensure_ascii=True, indent=2, sort_keys=True)) + return 0 if not report.get("errors") else 1 + except ( + OSError, + TypeError, + ValueError, + RealFeatureIngestionError, + m310.MetadataFabricSparkObjectStoreInteroperabilityError, + m310.identity.MetadataFabricGravitinoIdentityError, + m321.ObjectStoreActiveMetadataPromotionError, + ) as exc: + print(f"metadata fabric real feature ingestion: {exc}") + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/data_agent/platform_truth.py b/data_agent/platform_truth.py index a89e1c45..774817d4 100644 --- a/data_agent/platform_truth.py +++ b/data_agent/platform_truth.py @@ -998,6 +998,26 @@ def _config( ), "Protected object-store identity/TLS/KMS, real row ingestion and versioned promotion ledger", ), + RuntimeSpec( + "metadata_real_feature_ingestion_rehearsal", + "real_feature_ingestion_rehearsal", + "governed", + "evidence_durable", + "temporary authorized Spark/Sedona + JDBC/S3 ingestion and committed local evidence", + "metadata-platform", + "local_verification_only", + ( + "data_agent/metadata_fabric_real_feature_ingestion.py", + "scripts/metadata-fabric-real-feature-ingestion.sh", + ), + ( + ( + "data_agent/metadata_fabric_real_feature_ingestion.py", + "def run_live_rehearsal", + ), + ), + "Versioned GDA Control promotion, protected identity/storage and production ingestion", + ), RuntimeSpec( "datalake_monitor", "monitor_loop", diff --git a/data_agent/test_metadata_fabric_real_feature_ingestion.py b/data_agent/test_metadata_fabric_real_feature_ingestion.py new file mode 100644 index 00000000..413a238c --- /dev/null +++ b/data_agent/test_metadata_fabric_real_feature_ingestion.py @@ -0,0 +1,598 @@ +import inspect +import json +from copy import deepcopy +from datetime import UTC, datetime + +import geopandas as gpd +import pytest +from pydantic import SecretStr, ValidationError +from shapely.geometry import Polygon + +from data_agent import metadata_fabric_object_store_active_metadata_promotion as m321 +from data_agent import metadata_fabric_real_feature_ingestion as ingestion +from data_agent import metadata_fabric_spark_object_store_interoperability as m310 +from data_agent.platform_contracts import canonical_json_fingerprint + +AT = datetime(2026, 7, 31, 10, 0, tzinfo=UTC) +EXPECTED_EVIDENCE_SHA256 = ( + "42abd82613eaf28cb53c64280258bc75dba6cf841f9a513a4c801a9f798b9899" +) + + +def _profile(): + return ingestion.load_profile() + + +def _predecessor(): + profile = _profile() + return ingestion._load_json_object( + ingestion._resolve_repo_path(profile.dependencies.m321_evidence_path) + ) + + +def _source(): + dataset = _predecessor()["dataset_bundle"] + row_hashes = sorted( + canonical_json_fingerprint({"bounded_test_row": index}) for index in range(20) + ) + row_set_sha256 = canonical_json_fingerprint(row_hashes) + output_content_sha256 = ingestion._output_content_sha256( + source_content_sha256=dataset["content_sha256"], + row_set_sha256=row_set_sha256, + ) + return { + "inventory": dataset, + "projection": { + "schema": ingestion.ROW_SET_SCHEMA, + "feature_count": 20, + "unique_identifier_count": 20, + "valid_geometry_count": 20, + "non_empty_geometry_count": 20, + "geometry_z_count": 20, + "geometry_types": ["Polygon"], + "srid": 4490, + "bounds": [106.0, 29.0, 107.0, 30.0], + "row_set_sha256": row_set_sha256, + "row_sha256": row_hashes, + "payload_sha256": "1" * 64, + "payload_size_bytes": 100, + "source_payload_recorded": False, + }, + "payload": { + "source_content_sha256": dataset["content_sha256"], + "output_content_sha256": output_content_sha256, + "row_set_sha256": row_set_sha256, + }, + } + + +def _plan(): + return ingestion.build_ingestion_plan( + _profile(), + _predecessor(), + _source(), + { + "cluster_uid": "99999999-9999-4999-8999-999999999999", + "provider_runtime": "bounded-test-runtime", + }, + ) + + +def _spark(plan, authorization_sha256): + row_hashes = _source()["projection"]["row_sha256"] + snapshots = [{"snapshot_id": 71, "parent_id": None, "operation": "append"}] + data_files = [ + { + "file_path": ( + "s3://gda-metadata-warehouse/warehouse/cultural_heritage/" + "cultural_districts/data/00000-test.parquet" + ), + "record_count": 20, + } + ] + quality = { + "feature_count": 20, + "unique_bsm_count": 20, + "valid_geometry_count": 20, + "srid_match_count": 20, + "positive_area_count": 20, + "bbox_match_count": 20, + } + return { + "wait_completed": True, + "terminal_condition": "Complete", + "job": {"succeeded": 1, "failed": 0}, + "pod": { + "node_name": "desktop-worker", + "service_account": "spark-object-store-probe", + "service_account_automount_disabled": True, + "persistent_volume_claims": [], + }, + "result_line_count": 1, + "failure_diagnostic": [], + "result": { + "schema": ingestion.PROBE_RESULT_SCHEMA, + "plan_sha256": plan.ingestion_plan_sha256, + "authorization_sha256": authorization_sha256, + "source_resource_version_id": str(ingestion.SOURCE_RESOURCE_VERSION_ID), + "source_content_sha256": plan.source_content_sha256, + "output_resource_version_id": str(ingestion.OUTPUT_RESOURCE_VERSION_ID), + "output_content_sha256": plan.output_content_sha256, + "row_set_sha256": plan.row_set_sha256, + "spark_version": "3.5.0", + "sedona_version": "1.9.0", + "iceberg_runtime": "1.6.1", + "table_columns": list(ingestion.SPARK_COLUMNS), + "quality": quality, + "first_execution": { + "status": "appended", + "mutation_count": 1, + "row_sha256": row_hashes, + "snapshots": snapshots, + "data_files": data_files, + }, + "immediate_replay": { + "status": "no_op", + "mutation_count": 0, + "row_sha256": row_hashes, + "snapshots": snapshots, + "data_files": data_files, + }, + "source_payload_recorded": False, + "material_recorded": False, + }, + } + + +def _store(): + prefix = "warehouse/cultural_heritage/cultural_districts/" + data_key = f"{prefix}data/00000-test.parquet" + metadata_key = f"{prefix}metadata/00001-test.metadata.json" + manifest_key = f"{prefix}metadata/test-m0.avro" + return { + "bucket": "gda-metadata-warehouse", + "prefix": prefix, + "object_count": 3, + "objects": [ + {"key": data_key, "size": 2048, "etag": "data-etag"}, + {"key": metadata_key, "size": 1024, "etag": "metadata-etag"}, + {"key": manifest_key, "size": 512, "etag": "manifest-etag"}, + ], + "data_keys": [data_key], + "metadata_keys": [metadata_key], + "manifest_keys": [manifest_key], + "latest_metadata": { + "key": metadata_key, + "body_sha256": "2" * 64, + "size_bytes": 1024, + "location": _profile().target.table_location, + "current_snapshot_id": 71, + "current_schema_id": 0, + "fields": list(ingestion.ICEBERG_FIELDS), + }, + "source_feature_payload_recorded": False, + "material_recorded": False, + } + + +def test_profile_binds_checked_real_data_and_runtime_dependencies(): + profile = _profile() + predecessor, runtime_profile = ingestion._load_dependencies(profile) + + assert predecessor["evidence_sha256"] == ingestion.M321_EVIDENCE_SHA256 + assert predecessor["dataset_bundle"]["content_sha256"] == ( + "fd474fd65c8e4a71da241eb3fd07748ca3b972fbd2d3c32833376dbe71104007" + ) + assert runtime_profile.catalog.backend == "jdbc" + assert profile.source.expected_feature_count == 20 + assert profile.source.expected_srid == 4490 + assert profile.target.identity == ( + "gda_chongqing_m3_22/lakehouse/cultural_heritage/cultural_districts" + ) + assert profile.claims.production_ingestion_verified is False + + +def test_dependencies_use_full_predecessor_validators(monkeypatch): + m321_called = False + m310_called = False + m321_validator = m321.build_validation_report + m310_validator = m310.build_validation_report + + def checked_m321(**kwargs): + nonlocal m321_called + m321_called = True + return m321_validator(**kwargs) + + def checked_m310(**kwargs): + nonlocal m310_called + m310_called = True + return m310_validator(**kwargs) + + monkeypatch.setattr(m321, "build_validation_report", checked_m321) + monkeypatch.setattr(m310, "build_validation_report", checked_m310) + + ingestion._load_dependencies(_profile()) + + assert m321_called is True + assert m310_called is True + + +def test_source_input_is_deterministic_and_path_free(monkeypatch): + predecessor = _predecessor() + inventory = predecessor["dataset_bundle"] + geometries = [ + Polygon( + [ + (106 + index / 100, 29, 10), + (106.005 + index / 100, 29, 10), + (106.005 + index / 100, 29.005, 10), + (106 + index / 100, 29, 10), + ] + ) + for index in range(20) + ] + frame = gpd.GeoDataFrame( + {"Bsm": [f"bounded-{index:02d}" for index in range(20)]}, + geometry=geometries, + crs="EPSG:4490", + ) + monkeypatch.setattr( + ingestion, + "build_shapefile_bundle_inventory", + lambda *args, **kwargs: inventory, + ) + monkeypatch.setattr(ingestion.gpd, "read_file", lambda path: frame) + + source = ingestion.build_source_input( + _profile(), + predecessor, + shapefile_path=ingestion.Path("/private/source/real.shp"), + ogrinfo_path=ingestion.Path("/private/tool/ogrinfo"), + proj_data_path=None, + ) + + assert source["inventory"] == inventory + assert source["projection"]["feature_count"] == 20 + assert source["projection"]["geometry_z_count"] == 20 + assert len(set(source["projection"]["row_sha256"])) == 20 + assert source["payload"]["output_content_sha256"] == ( + ingestion._output_content_sha256( + source_content_sha256=inventory["content_sha256"], + row_set_sha256=source["projection"]["row_set_sha256"], + ) + ) + assert "/private/" not in json.dumps(source, sort_keys=True) + assert "geometry_wkb_hex" not in json.dumps(source["projection"], sort_keys=True) + + +def test_source_input_rejects_duplicate_identifiers(monkeypatch): + predecessor = _predecessor() + polygon = Polygon([(106, 29, 10), (107, 29, 10), (107, 30, 10), (106, 29, 10)]) + frame = gpd.GeoDataFrame( + {"Bsm": ["duplicate"] * 20}, geometry=[polygon] * 20, crs="EPSG:4490" + ) + monkeypatch.setattr( + ingestion, + "build_shapefile_bundle_inventory", + lambda *args, **kwargs: predecessor["dataset_bundle"], + ) + monkeypatch.setattr(ingestion.gpd, "read_file", lambda path: frame) + + with pytest.raises(ingestion.RealFeatureIngestionError): + ingestion.build_source_input( + _profile(), + predecessor, + shapefile_path=ingestion.Path("source.shp"), + ogrinfo_path=ingestion.Path("ogrinfo"), + proj_data_path=None, + ) + + +def test_plan_binds_source_rows_target_and_runtime(): + plan = _plan() + + assert plan.source_resource_version_id == ingestion.SOURCE_RESOURCE_VERSION_ID + assert plan.output_resource_version_id == ingestion.OUTPUT_RESOURCE_VERSION_ID + assert plan.predecessor_promotion_candidate_sha256 == ( + _predecessor()["promotion_candidate_sha256"] + ) + assert plan.runtime_binding_sha256 == canonical_json_fingerprint(plan.runtime_binding) + assert plan.table_columns == ingestion.GRAVITINO_COLUMNS + assert plan.writes_to_gda_control is False + + tampered = plan.model_dump(mode="json", by_alias=True) + tampered["row_set_sha256"] = "0" * 64 + with pytest.raises(ValidationError): + ingestion.RealFeatureIngestionPlan.model_validate(tampered) + + +def test_authorization_binds_the_exact_ingestion_plan(): + plan = _plan() + authorization = ingestion.build_ingestion_authorization( + plan, _profile(), authorized_at=AT + ) + + ingestion.validate_ingestion_authorization(plan, authorization, at=AT) + changed = list(authorization) + changed[-1] = "0" * 64 + with pytest.raises(ingestion.RealFeatureIngestionError): + ingestion.validate_ingestion_authorization(plan, tuple(changed), at=AT) + + +class _BoundedTableClient: + def __init__(self): + self.table = None + + def request(self, method, path, *, json_body=None, label): + if method == "POST": + self.table = deepcopy(json_body) + return 200, {"table": self.table} + assert method == "GET" + return 200, {"table": self.table} + + +class _TableRehearsal: + def __init__(self): + self.bounded = _BoundedTableClient() + + def _schema_path(self, target): + return "metalakes/test/catalogs/test/schemas/test" + + def _table_path(self, target): + return "metalakes/test/catalogs/test/schemas/test/tables/test" + + +def test_table_create_reads_source_binding_from_table_properties(): + rehearsal = _TableRehearsal() + + result = ingestion.create_target_table(rehearsal, _profile(), _plan()) + + assert result["source_binding_verified"] is True + assert result["mutation_count"] == 1 + properties = rehearsal.bounded.table["properties"] + assert properties["gda.source_resource_urn"] == _plan().source_resource_urn + assert properties["gda.row_set_sha256"] == _plan().row_set_sha256 + + +def test_ephemeral_large_input_uses_create_without_apply_annotation(): + source = inspect.getsource(ingestion._run_spark_ingestion) + + assert '["create", "-f", "-"]' in source + assert 'input_resource, ensure_ascii=True' in source + + +def test_spark_validation_requires_exact_authorization_and_no_op_replay(): + plan = _plan() + authorization_sha256 = "3" * 64 + spark = _spark(plan, authorization_sha256) + + assert ingestion._spark_errors( + spark, + plan, + _source(), + expected_authorization_sha256=authorization_sha256, + ) == [] + + tampered = deepcopy(spark) + tampered["result"]["authorization_sha256"] = "4" * 64 + errors = ingestion._spark_errors( + tampered, + plan, + _source(), + expected_authorization_sha256=authorization_sha256, + ) + assert "real feature Spark result is not authorization-bound" in errors + + +class _Body: + def __init__(self, payload): + self.payload = payload + + def read(self): + return self.payload + + +class _S3Client: + def __init__(self, metadata): + self.metadata = json.dumps(metadata).encode() + self.closed = False + + def list_objects_v2(self, **request): + assert request == { + "Bucket": "gda-metadata-warehouse", + "Prefix": "warehouse/cultural_heritage/cultural_districts/", + } + prefix = request["Prefix"] + return { + "IsTruncated": False, + "Contents": [ + {"Key": f"{prefix}data/00000-test.parquet", "Size": 2048}, + { + "Key": f"{prefix}metadata/00001-test.metadata.json", + "Size": len(self.metadata), + }, + {"Key": f"{prefix}metadata/test-m0.avro", "Size": 512}, + ], + } + + def get_object(self, **request): + assert request["Bucket"] == "gda-metadata-warehouse" + return {"Body": _Body(self.metadata)} + + def close(self): + self.closed = True + + +class _Runtime: + def __init__(self, client): + self.client = client + + def _s3_client(self, **kwargs): + assert kwargs["endpoint_url"] == "http://127.0.0.1:9000" + return self.client + + +def test_direct_s3_observation_requires_exact_ingested_table_metadata(): + metadata = { + "location": _profile().target.table_location, + "current-schema-id": 0, + "current-snapshot-id": 71, + "schemas": [ + { + "schema-id": 0, + "fields": [ + {"name": field["name"], "required": True, "type": field["type"]} + for field in ingestion.ICEBERG_FIELDS + ], + } + ], + } + client = _S3Client(metadata) + + observed = ingestion.observe_ingested_table( + _Runtime(client), + _profile(), + endpoint_url="http://127.0.0.1:9000", + object_store_user=SecretStr("user"), + object_store_material=SecretStr("material"), + ) + + assert len(observed["data_keys"]) == 1 + assert observed["latest_metadata"]["current_snapshot_id"] == 71 + assert observed["latest_metadata"]["fields"] == list(ingestion.ICEBERG_FIELDS) + assert client.closed is True + + +def test_object_store_validation_handles_missing_snapshot_without_crashing(): + spark = _spark(_plan(), "3" * 64) + spark["result"]["first_execution"]["snapshots"] = [] + + assert ingestion._object_store_errors(_store(), spark, _profile()) == [ + "direct S3 Iceberg data projection does not match Spark readback" + ] + + +def test_output_contracts_separate_quality_evaluator_and_lineage(): + plan = _plan() + contracts = ingestion.build_output_contracts( + plan, _spark(plan, "3" * 64), _store(), created_at=AT + ) + + assert contracts["output_resource_version"]["resource_version_id"] == str( + ingestion.OUTPUT_RESOURCE_VERSION_ID + ) + assert contracts["quality_result"]["evaluated_by"] == ingestion.QUALITY_EVALUATOR + assert contracts["quality_result"]["evaluated_by"] != ingestion.WORKLOAD + assert contracts["lineage_event"]["source_resource_version_id"] == str( + ingestion.SOURCE_RESOURCE_VERSION_ID + ) + assert contracts["lineage_event"]["target_resource_version_id"] == str( + ingestion.OUTPUT_RESOURCE_VERSION_ID + ) + assert contracts["persisted_to_gda_control"] is False + + +def _observation(): + plan = _plan() + authorization = ingestion.build_ingestion_authorization( + plan, _profile(), authorized_at=AT + ) + authorization_sha256 = authorization[-1] + spark = _spark(plan, authorization_sha256) + return { + "schema": ingestion.OBSERVATION_SCHEMA, + "observed_at": AT.isoformat(), + "contract": { + "contract_sha256": ingestion.build_contract_report()["contract_sha256"] + }, + "dataset_bundle": _source()["inventory"], + "source_projection": _source()["projection"], + "plan": plan.model_dump(mode="json", by_alias=True), + "authorization": { + "action": ingestion.ACTION, + "provider_apply_authorized": True, + "authorization_sha256": authorization_sha256, + }, + "table_create": { + "status": "created", + "mutation_count": 1, + "mutations": ["gravitino.table.create"], + "source_binding_verified": True, + }, + "spark": spark, + "object_store": _store(), + "output_contracts": ingestion.build_output_contracts( + plan, spark, _store(), created_at=AT + ), + "runtime_checks": { + "all_runtime_port_forwards_stopped": True, + "namespace_delete_completed": True, + "namespace_absent": True, + "persistent_volumes_absent": True, + "provider_objects_retained": False, + "object_store_objects_retained": False, + "material_recorded": False, + }, + } + + +def test_evidence_is_path_free_content_bound_and_fail_closed_for_production(): + evidence = ingestion.build_evidence(_observation(), profile=_profile()) + + assert evidence["errors"] == [] + assert ingestion.verify_evidence_integrity(evidence) == [] + assert evidence["local_real_feature_ingestion_verified"] is True + assert evidence["production_ingestion_verified"] is False + assert evidence["production_ready"] is False + serialized = json.dumps(evidence, sort_keys=True) + assert "/Users/" not in serialized + assert "geometry_wkb_hex" not in serialized + + tampered = _observation() + tampered["spark"]["result"]["authorization_sha256"] = "0" * 64 + rebuilt = ingestion.build_evidence(tampered, profile=_profile()) + assert "real feature Spark result is not authorization-bound" in rebuilt["errors"] + assert rebuilt["local_real_feature_ingestion_verified"] is False + + +def test_contract_is_valid_and_manifest_is_secret_free(): + report = ingestion.build_contract_report() + + assert report["status"] == "valid" + assert report["expected_feature_count"] == 20 + assert report["production_ingestion_verified"] is False + assert report["production_ready"] is False + assert ingestion._manifest_errors() == [] + + +def test_checked_evidence_is_valid_and_content_bound(): + evidence = ingestion._load_json_object(ingestion.DEFAULT_EVIDENCE_PATH) + report = ingestion.build_validation_report() + + assert evidence["evidence_sha256"] == EXPECTED_EVIDENCE_SHA256 + assert report["status"] == "valid" + assert report["errors"] == [] + assert report["local_real_feature_ingestion_verified"] is True + + tampered = deepcopy(evidence["observation"]) + tampered["object_store"]["latest_metadata"]["current_snapshot_id"] = -1 + rebuilt = ingestion.build_evidence(tampered, profile=_profile()) + assert "direct S3 Iceberg data projection does not match Spark readback" in rebuilt[ + "errors" + ] + + +def test_evidence_integrity_rejects_sensitive_material_and_overclaim(): + evidence = { + "schema": ingestion.EVIDENCE_SCHEMA, + "errors": [], + "local_real_feature_ingestion_verified": True, + "production_ingestion_verified": True, + "credential": "must-not-appear", + } + evidence["evidence_sha256"] = canonical_json_fingerprint(evidence) + + errors = ingestion.verify_evidence_integrity(evidence) + + assert "real feature evidence contains sensitive material" in errors + assert any("production_ingestion_verified" in error for error in errors) diff --git a/data_agent/test_platform_truth.py b/data_agent/test_platform_truth.py index 2a252622..f31b5445 100644 --- a/data_agent/test_platform_truth.py +++ b/data_agent/test_platform_truth.py @@ -292,6 +292,11 @@ def test_repository_source_access_and_runtime_baselines_match(): and item["production_role"] == "local_verification_only" for item in static_report["runtime"]["inventory"] ) + assert any( + item["runtime_id"] == "metadata_real_feature_ingestion_rehearsal" + and item["production_role"] == "local_verification_only" + for item in static_report["runtime"]["inventory"] + ) def test_runtime_report_detects_unregistered_background_mechanism(tmp_path): diff --git a/docs/architecture-decisions/adr-068-local-authorized-real-feature-iceberg-ingestion.md b/docs/architecture-decisions/adr-068-local-authorized-real-feature-iceberg-ingestion.md new file mode 100644 index 00000000..a4c2fadd --- /dev/null +++ b/docs/architecture-decisions/adr-068-local-authorized-real-feature-iceberg-ingestion.md @@ -0,0 +1,62 @@ +# ADR-068: Local authorized real-feature Iceberg ingestion + +**Status**: Accepted + +**Date**: 2026-07-31 + +**Decision owners**: Data Platform, Metadata Platform, Data Governance, GIS Platform, Security, Platform Architecture + +**Related decisions**: [ADR-056](adr-056-local-spark-object-store-interoperability.md) · [ADR-062](adr-062-atomic-active-metadata-authorization-and-dispatch.md) · [ADR-067](adr-067-object-store-runtime-bound-active-metadata-promotion.md) + +## Context + +M3-21 bound the real Chongqing cultural-district ResourceVersion to a local JDBC/S3 provider runtime, but deliberately created an empty Iceberg table. It proved metadata promotion and restart continuity, not ingestion of source feature rows. The next slice must exercise the real spatial payload without committing the source files, local paths, feature identifiers or geometry bytes. + +The source is the same 20-feature EPSG:4490 Shapefile bundle already content-bound in M3-16 and carried through M3-21. M3-22 must preserve that predecessor evidence, create an independent output ResourceVersion candidate and keep provider execution separate from GDA Control authority. + +## Decision + +### 1. Ingest one bounded real-data slice + +M3-22 reads only the checked Chongqing central cultural-district bundle. The local source path is a runtime argument and never enters the plan or committed evidence. The bundle SHA remains `fd474fd65c8e4a71da241eb3fd07748ca3b972fbd2d3c32833376dbe71104007`. + +Each input row is normalized to `BSM`, WKB geometry, SRID, four bounds and a canonical row SHA. The ephemeral payload is delivered through an immutable ConfigMap in a fresh namespace. Committed evidence keeps only the bundle inventory, aggregate spatial projection, row hashes and payload hash; it excludes BSM values, WKB and absolute paths. + +### 2. Bind authorization to the exact source, output and runtime + +The execution-plan Artifact binds source ResourceVersion `a6000000-0000-4000-8000-000000000001`, output ResourceVersion `a6000000-0000-4000-8000-000000000002`, source content SHA, row-set SHA, output content SHA, M3-21 predecessor candidate, target schema and stable JDBC/S3 runtime identity. + +One allow PolicyDecision and independent ApprovalRecord authorize `metadata_fabric.ingest_real_feature_slice`. The Spark result must return the exact authorization fingerprint. The table is created by the schema-bounded Gravitino principal; catalog administration remains denied with `403`. + +### 3. Require spatial quality, one append and exact replay + +One Spark `3.5.0` + Sedona `1.9.0` Job reconstructs geometry through `ST_GeomFromWKB` and verifies all 20 rows for unique identifier, valid geometry, EPSG:4490, positive area and source-matching bounds. It writes the fixed eight-column schema through Iceberg `1.6.1`. + +The first execution must append exactly once and produce one snapshot and one Parquet file. An immediate execution of the same plan must be `no_op/0` with identical row hashes, snapshot and data-file projections. Direct S3 inspection must agree with the Spark readback and exact Iceberg schema. + +### 4. Keep local evidence non-authoritative + +The output ResourceVersion, Artifact, independent QualityResult and LineageEvent are candidates only. They are not written to GDA Control, and the PlatformRun is not finalized. The local MinIO/JDBC runtime uses generated static material over HTTP on one Docker Desktop host and is deleted after the rehearsal. + +Therefore protected workload identity, durable production catalog, production object storage, OIDC, TLS, complete Spark/Flink conformance, production ingestion, PlatformRun success and `production_ready` remain false. + +## Verification + +The local rehearsal recorded: + +- 20 unique, valid, non-empty Polygon/MultiPolygon Z features in EPSG:4490; +- row-set SHA `c26ff708f4b6be082327dff63a6a8659420dbc4cab37dea1cac7b40f147512df` and output content SHA `bdc06792e8b935176ee6df6f6f6d4be1535622d54d9b994a778cabfe5a574618`; +- six Sedona quality counts equal to 20; +- first execution `appended/1`, one snapshot, one 20-row Parquet file, followed by immediate `no_op/0`; +- direct S3 inventory of one data file, two metadata JSON files and two Avro manifests, with current snapshot and eight fields matching Spark; +- path-free output ResourceVersion, Artifact, independent passed QualityResult and source-to-output LineageEvent candidates; +- contract SHA `af211f2d2f4830decb9ffe369cd9e7ec2c9349c2e2c8bd789347a6fdc288e1dc` and evidence SHA `42abd82613eaf28cb53c64280258bc75dba6cf841f9a513a4c801a9f798b9899`; +- complete namespace, PVC and port-forward cleanup. + +## Consequences + +**Positive**: the platform now has checked evidence that a real, content-bound GIS slice can cross authorization, spatial quality, Spark/Sedona execution, Iceberg storage, direct object inspection and path-free version/quality/lineage construction. + +**Negative**: the evidence covers only 20 features in a temporary local environment. It does not prove large-volume partitioning, schema evolution for heterogeneous sources, concurrent ingestion, recovery after process loss, production identity/storage or authoritative publication. + +**Next gate**: atomically promote the output ResourceVersion, Artifact, QualityResult and LineageEvent candidates through the versioned GDA Control ledger while preserving a non-success Run until the existing terminal evidence gate is satisfied. Production promotion still requires protected identity/TLS/KMS, selected object storage, independent failure domains, backup/PITR, tenant isolation and staging-scale verification. diff --git a/docs/evidence/metadata-fabric-real-feature-ingestion-2026-07-31.json b/docs/evidence/metadata-fabric-real-feature-ingestion-2026-07-31.json new file mode 100644 index 00000000..ec0a91e8 --- /dev/null +++ b/docs/evidence/metadata-fabric-real-feature-ingestion-2026-07-31.json @@ -0,0 +1,1022 @@ +{ + "authorized_spark_execution_verified": true, + "contract_sha256": "af211f2d2f4830decb9ffe369cd9e7ec2c9349c2e2c8bd789347a6fdc288e1dc", + "dataset_bundle": { + "components": [ + { + "component": ".cpg", + "sha256": "3ad3031f5503a4404af825262ee8232cc04d4ea6683d42c5dd0a2f2a27ac9824", + "size_bytes": 5 + }, + { + "component": ".dbf", + "sha256": "ee7c6c4c6957aea296b69d62118d416e5ee989aa77f7b98cf0fe580874ce5127", + "size_bytes": 44990 + }, + { + "component": ".prj", + "sha256": "b10dbe4d6d1de908d340f892c90b3d31a552630af3742bb515bfe1bd26124f2c", + "size_bytes": 176 + }, + { + "component": ".sbn", + "sha256": "7d0279465b18beec40308717e0ef0ea5701a586bc5c84e6a9aa309d5bc0ec99a", + "size_bytes": 308 + }, + { + "component": ".sbx", + "sha256": "019156149b2c7771ec0dd249c757dd7aa01a98075e246a77f77b080860c57333", + "size_bytes": 124 + }, + { + "component": ".shp", + "sha256": "6ac0d5c8c8db66fc0e2a74d8232b7779bd2454257df14efa2930e3dbc181aed0", + "size_bytes": 283640 + }, + { + "component": ".shp.xml", + "sha256": "8ef222ce1952552b366acf14a996e1c8cbdbe3eed0bb829dacfe7eafd068d948", + "size_bytes": 43100 + }, + { + "component": ".shx", + "sha256": "f3fbb6a7775ca833c066e3a3f2a332f99f979840045909ae187d08dac126a119", + "size_bytes": 260 + } + ], + "content_sha256": "fd474fd65c8e4a71da241eb3fd07748ca3b972fbd2d3c32833376dbe71104007", + "format": "ESRI Shapefile", + "schema": "gda.spatial_dataset_bundle.v1", + "source_label": "chongqing-central-cultural-districts", + "spatial_inventory": { + "bounds": [ + 106.37987914500007, + 29.558008447000077, + 106.59532712300008, + 29.877271985000025 + ], + "crs": { + "authority": "EPSG", + "code": 4490, + "name": "China Geodetic Coordinate System 2000" + }, + "driver": "ESRI Shapefile", + "feature_count": 20, + "field_count": 33, + "geometry_type": "PolygonZ" + } + }, + "direct_object_store_data_verified": true, + "durable_catalog_verified": false, + "environment": "local_docker_desktop", + "errors": [], + "evidence_sha256": "42abd82613eaf28cb53c64280258bc75dba6cf841f9a513a4c801a9f798b9899", + "exact_ingestion_replay_no_op_verified": true, + "flink_conformance_verified": false, + "iceberg_single_snapshot_verified": true, + "ingestion_persisted_to_gda_control": false, + "local_real_feature_ingestion_verified": true, + "m310_evidence_fingerprint": "05844457efb378581fb7fc2e7ed3c706819b2d8fa5a52b2f82577051d38c2cd1", + "m321_evidence_sha256": "d73754c53cf16d888aa345baa5d079cc7fd98d8b84db747f52188c1a69bf1628", + "observation": { + "authorization": { + "action": "metadata_fabric.ingest_real_feature_slice", + "approval_artifact_id": "6a8169f5-4f71-5132-8594-9d24bda77cb0", + "authorization_sha256": "7eb8bfae2306221e1f816d139a5347df8d3b7107058417cebb10e038f2175d9c", + "execution_plan_artifact_id": "486ac882-31c9-5afe-859f-81629b3298fe", + "policy_decision_artifact_id": "89e759f9-de00-5bf7-ae1c-1add3d9fa0a8", + "provider_apply_authorized": true, + "run_id": "a9000000-0000-4000-8000-000000000009" + }, + "bootstrap": { + "admin_authentication_status": 200, + "bounded_authentication_status": 200, + "bucket": "gda-metadata-warehouse", + "catalog": "lakehouse", + "catalog_backend": "jdbc", + "catalog_uri": "jdbc:postgresql://gravitino-persistence-postgresql:5432/iceberg", + "denied_catalog_create_status": 403, + "io_impl": "org.apache.iceberg.aws.s3.S3FileIO", + "material_recorded": false, + "metalake": "gda_chongqing_m3_22", + "role": { + "name": "gda-object-store-cultural-district-projector", + "securable_objects": [ + { + "fullName": "lakehouse", + "privileges": [ + { + "condition": "ALLOW", + "name": "USE_CATALOG" + } + ], + "type": "CATALOG" + }, + { + "fullName": "lakehouse.cultural_heritage", + "privileges": [ + { + "condition": "ALLOW", + "name": "CREATE_TABLE" + }, + { + "condition": "ALLOW", + "name": "USE_SCHEMA" + } + ], + "type": "SCHEMA" + } + ] + }, + "s3_endpoint": "http://metadata-object-store:9000", + "s3_path_style_access": true, + "s3_region": "us-east-1", + "schema": "cultural_heritage", + "server_version": "1.3.0", + "warehouse": "s3://gda-metadata-warehouse/warehouse" + }, + "cluster_uid": "c3c9bbab-2b36-4359-a7ac-e3de194445ab", + "contract": { + "contract_sha256": "af211f2d2f4830decb9ffe369cd9e7ec2c9349c2e2c8bd789347a6fdc288e1dc", + "m310_evidence_fingerprint": "05844457efb378581fb7fc2e7ed3c706819b2d8fa5a52b2f82577051d38c2cd1", + "m321_evidence_sha256": "d73754c53cf16d888aa345baa5d079cc7fd98d8b84db747f52188c1a69bf1628" + }, + "dataset_bundle": { + "components": [ + { + "component": ".cpg", + "sha256": "3ad3031f5503a4404af825262ee8232cc04d4ea6683d42c5dd0a2f2a27ac9824", + "size_bytes": 5 + }, + { + "component": ".dbf", + "sha256": "ee7c6c4c6957aea296b69d62118d416e5ee989aa77f7b98cf0fe580874ce5127", + "size_bytes": 44990 + }, + { + "component": ".prj", + "sha256": "b10dbe4d6d1de908d340f892c90b3d31a552630af3742bb515bfe1bd26124f2c", + "size_bytes": 176 + }, + { + "component": ".sbn", + "sha256": "7d0279465b18beec40308717e0ef0ea5701a586bc5c84e6a9aa309d5bc0ec99a", + "size_bytes": 308 + }, + { + "component": ".sbx", + "sha256": "019156149b2c7771ec0dd249c757dd7aa01a98075e246a77f77b080860c57333", + "size_bytes": 124 + }, + { + "component": ".shp", + "sha256": "6ac0d5c8c8db66fc0e2a74d8232b7779bd2454257df14efa2930e3dbc181aed0", + "size_bytes": 283640 + }, + { + "component": ".shp.xml", + "sha256": "8ef222ce1952552b366acf14a996e1c8cbdbe3eed0bb829dacfe7eafd068d948", + "size_bytes": 43100 + }, + { + "component": ".shx", + "sha256": "f3fbb6a7775ca833c066e3a3f2a332f99f979840045909ae187d08dac126a119", + "size_bytes": 260 + } + ], + "content_sha256": "fd474fd65c8e4a71da241eb3fd07748ca3b972fbd2d3c32833376dbe71104007", + "format": "ESRI Shapefile", + "schema": "gda.spatial_dataset_bundle.v1", + "source_label": "chongqing-central-cultural-districts", + "spatial_inventory": { + "bounds": [ + 106.37987914500007, + 29.558008447000077, + 106.59532712300008, + 29.877271985000025 + ], + "crs": { + "authority": "EPSG", + "code": 4490, + "name": "China Geodetic Coordinate System 2000" + }, + "driver": "ESRI Shapefile", + "feature_count": 20, + "field_count": 33, + "geometry_type": "PolygonZ" + } + }, + "initial_runtime": { + "context": "docker-desktop", + "gravitino": { + "image": "docker.io/gda/gravitino:1.3.0-local-arm64", + "image_id": "sha256:18e24b43be854dabdc13e96b1019eb3dc691d59cc64e411aa6a3cc49225fe2d3", + "node_name": "desktop-worker", + "persistent_volume_claims": [], + "pod_name": "gravitino-persistence-0", + "pod_uid": "089a6d11-4842-4836-a9f4-f3628b95775b", + "pvc": null, + "ready_replicas": 1, + "service_account": "gravitino-persistence", + "service_account_automount_disabled": true, + "statefulset_uid": "1353b086-efbb-4faf-bd3d-a9904343ac2c" + }, + "gravitino_aws_sdk_mounted": true, + "gravitino_host_image_id": "sha256:d355dc7e92f9e3545d717f3eab2cbdf412115f2b82e1e544d7f6235c1eacd5a5", + "gravitino_jdbc_driver_mounted": true, + "iceberg_rest": { + "aws_sdk_mounted": true, + "image": "docker.io/gda/gravitino:1.3.0-local-arm64", + "image_id": "sha256:18e24b43be854dabdc13e96b1019eb3dc691d59cc64e411aa6a3cc49225fe2d3", + "jdbc_driver_mounted": true, + "path": "/iceberg", + "ready": true + }, + "minio_host_image_id": "sha256:a1ea29fa28355559ef137d71fc570e508a214ec84ff8083e39bc5428980b015e", + "namespace": { + "name": "gda-metadata-spark-object-store", + "uid": "d48b10a6-d144-4983-b988-e68def2172a8" + }, + "object_store": { + "image": "docker.io/minio/minio:RELEASE.2025-04-22T22-12-26Z", + "image_id": "docker.io/minio/minio@sha256:a1ea29fa28355559ef137d71fc570e508a214ec84ff8083e39bc5428980b015e", + "node_name": "desktop-control-plane", + "persistent_volume_claims": [ + "data-metadata-object-store-0" + ], + "pod_name": "metadata-object-store-0", + "pod_uid": "42c60720-b3b2-45d4-bee0-7a24160f2d3f", + "pvc": { + "name": "data-metadata-object-store-0", + "phase": "Bound", + "storage_class": "standard", + "uid": "360086c0-7a38-40b3-a959-576764a5e17b", + "volume_name": "pvc-360086c0-7a38-40b3-a959-576764a5e17b" + }, + "ready_replicas": 1, + "service_account": "metadata-object-store", + "service_account_automount_disabled": true, + "statefulset_uid": "00141278-7192-470c-825b-4a2111504014" + }, + "object_store_service": { + "name": "metadata-object-store", + "ports": [ + { + "name": "api", + "port": 9000 + } + ], + "type": "ClusterIP", + "uid": "1f285fe0-734d-48e5-bf65-5512659b1425" + }, + "postgresql": { + "image": "docker.io/library/postgres:16.10-bookworm", + "image_id": "docker.io/library/postgres@sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74", + "node_name": "desktop-worker", + "persistent_volume_claims": [ + "data-gravitino-persistence-postgresql-0" + ], + "pod_name": "gravitino-persistence-postgresql-0", + "pod_uid": "da1719c7-66e7-4f05-ae2c-b93ee7a6dff4", + "pvc": { + "name": "data-gravitino-persistence-postgresql-0", + "phase": "Bound", + "storage_class": "standard", + "uid": "a525f752-aa0a-48bb-84af-7a7996b6d280", + "volume_name": "pvc-a525f752-aa0a-48bb-84af-7a7996b6d280" + }, + "ready_replicas": 1, + "service_account": "gravitino-persistence-postgresql", + "service_account_automount_disabled": true, + "statefulset_uid": "afe740f4-5f47-47f6-9ea7-8ee9b18bdfa3" + }, + "service": { + "name": "gravitino-persistence", + "ports": [ + { + "name": "http", + "port": 8090 + }, + { + "name": "iceberg-rest", + "port": 9001 + } + ], + "type": "ClusterIP", + "uid": "cef7ef5d-92ff-4cb5-acf8-042bf3f3b63f" + }, + "source_schema_sha256": "7a2d605a677a462ca619dba594ce7ebcf500358345560ad084c1b67a25c722df", + "spark_host_image_id": "sha256:f201367640c7583add224796a629150e63d3859ddd7fe9fd47741662a6d415bb" + }, + "object_store": { + "bucket": "gda-metadata-warehouse", + "data_keys": [ + "warehouse/cultural_heritage/cultural_districts/data/00000-8-11851edc-05b2-4ce0-9a1c-17b7ef344b4d-0-00001.parquet" + ], + "latest_metadata": { + "body_sha256": "7806fe74a812ca716ee42d33bd6b51aba4673d12d2244f0a736d052a8fdb4503", + "current_schema_id": 0, + "current_snapshot_id": 6084664108418947049, + "fields": [ + { + "name": "BSM", + "required": true, + "type": "string" + }, + { + "name": "geometry", + "required": true, + "type": "binary" + }, + { + "name": "srid", + "required": true, + "type": "int" + }, + { + "name": "min_x", + "required": true, + "type": "double" + }, + { + "name": "min_y", + "required": true, + "type": "double" + }, + { + "name": "max_x", + "required": true, + "type": "double" + }, + { + "name": "max_y", + "required": true, + "type": "double" + }, + { + "name": "row_sha256", + "required": true, + "type": "string" + } + ], + "key": "warehouse/cultural_heritage/cultural_districts/metadata/00001-f5fb2587-7669-4380-93cf-c2f78fffad94.metadata.json", + "location": "s3://gda-metadata-warehouse/warehouse/cultural_heritage/cultural_districts", + "size_bytes": 3424 + }, + "manifest_keys": [ + "warehouse/cultural_heritage/cultural_districts/metadata/0b8023c0-8240-4ea4-9b35-8eb39dce609f-m0.avro", + "warehouse/cultural_heritage/cultural_districts/metadata/snap-6084664108418947049-1-0b8023c0-8240-4ea4-9b35-8eb39dce609f.avro" + ], + "material_recorded": false, + "metadata_keys": [ + "warehouse/cultural_heritage/cultural_districts/metadata/00000-c2bfa142-e4eb-4fca-afcd-908a13ab3edd.metadata.json", + "warehouse/cultural_heritage/cultural_districts/metadata/00001-f5fb2587-7669-4380-93cf-c2f78fffad94.metadata.json" + ], + "object_count": 5, + "object_inventory_sha256": "032dd799082946a8c3cdc7e3267f0b6b5265a18ce406f30123fdedcd69068aca", + "objects": [ + { + "etag": "dd4e0f599837c5926ba4ec69d9c7371c", + "key": "warehouse/cultural_heritage/cultural_districts/data/00000-8-11851edc-05b2-4ce0-9a1c-17b7ef344b4d-0-00001.parquet", + "size": 94603 + }, + { + "etag": "bd798071453afef7d7b1efa75fb23d7f", + "key": "warehouse/cultural_heritage/cultural_districts/metadata/00000-c2bfa142-e4eb-4fca-afcd-908a13ab3edd.metadata.json", + "size": 2333 + }, + { + "etag": "60b3ff46145f0457349a03d65b0ca2f9", + "key": "warehouse/cultural_heritage/cultural_districts/metadata/00001-f5fb2587-7669-4380-93cf-c2f78fffad94.metadata.json", + "size": 3424 + }, + { + "etag": "d2fe842c48b1b18860635043d342f4e5", + "key": "warehouse/cultural_heritage/cultural_districts/metadata/0b8023c0-8240-4ea4-9b35-8eb39dce609f-m0.avro", + "size": 7613 + }, + { + "etag": "e11682b00b6863d330f612f65bd58cd8", + "key": "warehouse/cultural_heritage/cultural_districts/metadata/snap-6084664108418947049-1-0b8023c0-8240-4ea4-9b35-8eb39dce609f.avro", + "size": 4465 + } + ], + "prefix": "warehouse/cultural_heritage/cultural_districts/", + "source_feature_payload_recorded": false + }, + "object_store_prepared": { + "bucket": "gda-metadata-warehouse", + "created": true, + "head_bucket_verified": true, + "material_recorded": false, + "path_style_access": true, + "region": "us-east-1", + "service": "metadata-object-store" + }, + "observed_at": "2026-07-31T01:16:51.085417+00:00", + "output_contracts": { + "lineage_event": { + "artifact_id": "1f73db93-92e7-5970-842f-2477d6de394b", + "definition_version_id": "a9000000-0000-4000-8000-000000000008", + "event_sha256": "9269b95ef3d5400628939520ed9ada93cf1801db6a7ad1880f2ebdc996db88dc", + "event_type": "derive", + "facets": { + "feature_count": 20, + "row_set_sha256": "c26ff708f4b6be082327dff63a6a8659420dbc4cab37dea1cac7b40f147512df", + "snapshot_id": 6084664108418947049 + }, + "lineage_event_id": "06f11901-1623-564b-bcd6-389808c774f2", + "occurred_at": "2026-07-31T01:15:33.501911Z", + "producer": "workload:real-feature-ingestion-executor", + "run_id": "a9000000-0000-4000-8000-000000000009", + "source_resource_version_id": "a6000000-0000-4000-8000-000000000001", + "target_resource_version_id": "a6000000-0000-4000-8000-000000000002", + "tenant_id": "metadata-authorization-local" + }, + "output_artifact": { + "artifact_id": "1f73db93-92e7-5970-842f-2477d6de394b", + "artifact_key": "cultural-districts-iceberg:1f73db93-92e7-5970-842f-2477d6de394b", + "artifact_role": "output", + "content_sha256": "bdc06792e8b935176ee6df6f6f6d4be1535622d54d9b994a778cabfe5a574618", + "created_at": "2026-07-31T01:15:33.501911Z", + "created_by": "workload:real-feature-ingestion-executor", + "manifest": { + "data_file_count": 1, + "feature_count": 20, + "row_set_sha256": "c26ff708f4b6be082327dff63a6a8659420dbc4cab37dea1cac7b40f147512df", + "snapshot_id": 6084664108418947049 + }, + "media_type": "application/vnd.apache.iceberg.table", + "resource_version_id": "a6000000-0000-4000-8000-000000000002", + "run_id": "a9000000-0000-4000-8000-000000000009", + "size_bytes": 94603, + "storage_uri": "s3://gda-metadata-warehouse/warehouse/cultural_heritage/cultural_districts", + "tenant_id": "metadata-authorization-local" + }, + "output_resource_version": { + "authority_version_ref": { + "catalog": "lakehouse", + "row_set_sha256": "c26ff708f4b6be082327dff63a6a8659420dbc4cab37dea1cac7b40f147512df", + "schema": "cultural_heritage", + "snapshot_id": 6084664108418947049, + "table": "cultural_districts" + }, + "content_sha256": "bdc06792e8b935176ee6df6f6f6d4be1535622d54d9b994a778cabfe5a574618", + "created_at": "2026-07-31T01:15:33.501911Z", + "created_by": "workload:real-feature-ingestion-executor", + "predecessor_version_id": null, + "resource_urn": "gda://metadata-authorization-local/data_product/chongqing-cultural-districts-iceberg", + "resource_version_id": "a6000000-0000-4000-8000-000000000002", + "tenant_id": "metadata-authorization-local", + "version_key": "rows-bdc06792e8b9" + }, + "persisted_to_gda_control": false, + "quality_evidence_artifact": { + "artifact_id": "bd85b0cb-df7e-5d90-8f3f-585664cd1188", + "artifact_key": "real-feature-quality:bd85b0cb-df7e-5d90-8f3f-585664cd1188", + "artifact_role": "evidence", + "content_sha256": "7806fe74a812ca716ee42d33bd6b51aba4673d12d2244f0a736d052a8fdb4503", + "created_at": "2026-07-31T01:15:33.501911Z", + "created_by": "workload:real-feature-ingestion-executor", + "manifest": { + "metrics": { + "bbox_match_count": 20, + "feature_count": 20, + "positive_area_count": 20, + "srid_match_count": 20, + "unique_bsm_count": 20, + "valid_geometry_count": 20 + }, + "row_set_sha256": "c26ff708f4b6be082327dff63a6a8659420dbc4cab37dea1cac7b40f147512df", + "rule_version_ref": "quality://gda/spatial/real-feature-ingestion/v1" + }, + "media_type": "application/vnd.apache.iceberg+json", + "resource_version_id": "a6000000-0000-4000-8000-000000000002", + "run_id": "a9000000-0000-4000-8000-000000000009", + "size_bytes": 3424, + "storage_uri": "s3://gda-metadata-warehouse/warehouse/cultural_heritage/cultural_districts/metadata/00001-f5fb2587-7669-4380-93cf-c2f78fffad94.metadata.json", + "tenant_id": "metadata-authorization-local" + }, + "quality_result": { + "evaluated_at": "2026-07-31T01:15:34.501911Z", + "evaluated_by": "workload:real-feature-spatial-quality-evaluator", + "evidence_artifact_id": "bd85b0cb-df7e-5d90-8f3f-585664cd1188", + "metrics": { + "bbox_match_count": 20, + "feature_count": 20, + "positive_area_count": 20, + "srid_match_count": 20, + "unique_bsm_count": 20, + "valid_geometry_count": 20 + }, + "quality_result_id": "52d38e8a-181f-5d16-8948-8432d3113f06", + "resource_version_id": "a6000000-0000-4000-8000-000000000002", + "result_sha256": "6a68a7c832ac6c2de23a43f0049cdb9cb2c808c3e8e289a093483be967b19635", + "rule_version_ref": "quality://gda/spatial/real-feature-ingestion/v1", + "run_id": "a9000000-0000-4000-8000-000000000009", + "tenant_id": "metadata-authorization-local", + "verdict": "passed" + } + }, + "plan": { + "definition_version_id": "a9000000-0000-4000-8000-000000000008", + "expected_feature_count": 20, + "ingestion_plan_sha256": "6282e636cc9399e5f8ddc41defb2123a2a31d1d2ae925e3b3660c48198dfbf84", + "output_content_sha256": "bdc06792e8b935176ee6df6f6f6d4be1535622d54d9b994a778cabfe5a574618", + "output_resource_urn": "gda://metadata-authorization-local/data_product/chongqing-cultural-districts-iceberg", + "output_resource_version_id": "a6000000-0000-4000-8000-000000000002", + "predecessor_promotion_candidate_sha256": "63812c311b3f239bc6a944748c4ff384250eb9c9ed9009d3384fc699f1d3eaa9", + "row_set_sha256": "c26ff708f4b6be082327dff63a6a8659420dbc4cab37dea1cac7b40f147512df", + "run_id": "a9000000-0000-4000-8000-000000000009", + "runtime_binding": { + "catalog": { + "backend": "jdbc", + "bucket": "gda-metadata-warehouse", + "io_impl": "org.apache.iceberg.aws.s3.S3FileIO", + "s3_endpoint": "http://metadata-object-store:9000", + "s3_path_style_access": true, + "s3_region": "us-east-1", + "uri": "jdbc:postgresql://gravitino-persistence-postgresql:5432/iceberg", + "warehouse": "s3://gda-metadata-warehouse/warehouse" + }, + "cluster_uid": "c3c9bbab-2b36-4359-a7ac-e3de194445ab", + "context": "docker-desktop", + "images": { + "gravitino_image_id": "sha256:18e24b43be854dabdc13e96b1019eb3dc691d59cc64e411aa6a3cc49225fe2d3", + "object_store_image_id": "docker.io/minio/minio@sha256:a1ea29fa28355559ef137d71fc570e508a214ec84ff8083e39bc5428980b015e", + "postgresql_image_id": "docker.io/library/postgres@sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74" + }, + "namespace": { + "name": "gda-metadata-spark-object-store", + "uid": "d48b10a6-d144-4983-b988-e68def2172a8" + }, + "schema": "gda.object_store_provider_runtime_binding.v1", + "services": { + "gravitino": { + "name": "gravitino-persistence", + "uid": "cef7ef5d-92ff-4cb5-acf8-042bf3f3b63f" + }, + "object_store": { + "name": "metadata-object-store", + "uid": "1f285fe0-734d-48e5-bf65-5512659b1425" + } + }, + "storage": { + "gravitino_persistent_volume_claims": [], + "object_store_pvc_uid": "360086c0-7a38-40b3-a959-576764a5e17b", + "object_store_volume_name": "pvc-360086c0-7a38-40b3-a959-576764a5e17b", + "postgresql_pvc_uid": "a525f752-aa0a-48bb-84af-7a7996b6d280", + "postgresql_volume_name": "pvc-a525f752-aa0a-48bb-84af-7a7996b6d280" + }, + "topology": { + "object_store_node": "desktop-control-plane", + "provider_node": "desktop-worker" + }, + "workloads": { + "gravitino_statefulset_uid": "1353b086-efbb-4faf-bd3d-a9904343ac2c", + "object_store_statefulset_uid": "00141278-7192-470c-825b-4a2111504014", + "postgresql_statefulset_uid": "afe740f4-5f47-47f6-9ea7-8ee9b18bdfa3" + } + }, + "runtime_binding_sha256": "556bf74f97f91c6ca3910e5e51175ae1ce8fec125b7bb17a341bd61447c444b7", + "schema": "gda.real_feature_ingestion_plan.v1", + "source_content_sha256": "fd474fd65c8e4a71da241eb3fd07748ca3b972fbd2d3c32833376dbe71104007", + "source_resource_urn": "gda://metadata-authorization-local/dataset/chongqing-cultural-districts", + "source_resource_version_id": "a6000000-0000-4000-8000-000000000001", + "table_columns": [ + { + "name": "BSM", + "nullable": false, + "type": "string" + }, + { + "name": "geometry", + "nullable": false, + "type": "binary" + }, + { + "name": "srid", + "nullable": false, + "type": "integer" + }, + { + "name": "min_x", + "nullable": false, + "type": "double" + }, + { + "name": "min_y", + "nullable": false, + "type": "double" + }, + { + "name": "max_x", + "nullable": false, + "type": "double" + }, + { + "name": "max_y", + "nullable": false, + "type": "double" + }, + { + "name": "row_sha256", + "nullable": false, + "type": "string" + } + ], + "target": { + "bucket": "gda-metadata-warehouse", + "catalog": "lakehouse", + "catalog_backend": "jdbc", + "catalog_provider": "lakehouse-iceberg", + "catalog_type": "RELATIONAL", + "io_impl": "org.apache.iceberg.aws.s3.S3FileIO", + "jdbc_driver": "org.postgresql.Driver", + "metalake": "gda_chongqing_m3_22", + "object_prefix": "warehouse/cultural_heritage/cultural_districts/", + "s3_endpoint": "http://metadata-object-store:9000", + "s3_path_style_access": true, + "s3_region": "us-east-1", + "schema": "cultural_heritage", + "table": "cultural_districts", + "uri": "jdbc:postgresql://gravitino-persistence-postgresql:5432/iceberg", + "warehouse": "s3://gda-metadata-warehouse/warehouse" + }, + "tenant_id": "metadata-authorization-local", + "writes_to_gda_control": false, + "writes_to_legacy": false + }, + "runtime_binding": { + "catalog": { + "backend": "jdbc", + "bucket": "gda-metadata-warehouse", + "io_impl": "org.apache.iceberg.aws.s3.S3FileIO", + "s3_endpoint": "http://metadata-object-store:9000", + "s3_path_style_access": true, + "s3_region": "us-east-1", + "uri": "jdbc:postgresql://gravitino-persistence-postgresql:5432/iceberg", + "warehouse": "s3://gda-metadata-warehouse/warehouse" + }, + "cluster_uid": "c3c9bbab-2b36-4359-a7ac-e3de194445ab", + "context": "docker-desktop", + "images": { + "gravitino_image_id": "sha256:18e24b43be854dabdc13e96b1019eb3dc691d59cc64e411aa6a3cc49225fe2d3", + "object_store_image_id": "docker.io/minio/minio@sha256:a1ea29fa28355559ef137d71fc570e508a214ec84ff8083e39bc5428980b015e", + "postgresql_image_id": "docker.io/library/postgres@sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74" + }, + "namespace": { + "name": "gda-metadata-spark-object-store", + "uid": "d48b10a6-d144-4983-b988-e68def2172a8" + }, + "schema": "gda.object_store_provider_runtime_binding.v1", + "services": { + "gravitino": { + "name": "gravitino-persistence", + "uid": "cef7ef5d-92ff-4cb5-acf8-042bf3f3b63f" + }, + "object_store": { + "name": "metadata-object-store", + "uid": "1f285fe0-734d-48e5-bf65-5512659b1425" + } + }, + "storage": { + "gravitino_persistent_volume_claims": [], + "object_store_pvc_uid": "360086c0-7a38-40b3-a959-576764a5e17b", + "object_store_volume_name": "pvc-360086c0-7a38-40b3-a959-576764a5e17b", + "postgresql_pvc_uid": "a525f752-aa0a-48bb-84af-7a7996b6d280", + "postgresql_volume_name": "pvc-a525f752-aa0a-48bb-84af-7a7996b6d280" + }, + "topology": { + "object_store_node": "desktop-control-plane", + "provider_node": "desktop-worker" + }, + "workloads": { + "gravitino_statefulset_uid": "1353b086-efbb-4faf-bd3d-a9904343ac2c", + "object_store_statefulset_uid": "00141278-7192-470c-825b-4a2111504014", + "postgresql_statefulset_uid": "afe740f4-5f47-47f6-9ea7-8ee9b18bdfa3" + } + }, + "runtime_binding_sha256": "556bf74f97f91c6ca3910e5e51175ae1ce8fec125b7bb17a341bd61447c444b7", + "runtime_checks": { + "all_runtime_port_forwards_stopped": true, + "material_recorded": false, + "namespace_absent": true, + "namespace_delete_completed": true, + "object_store_objects_retained": false, + "persistent_volume_names": [ + "pvc-360086c0-7a38-40b3-a959-576764a5e17b", + "pvc-a525f752-aa0a-48bb-84af-7a7996b6d280" + ], + "persistent_volumes_absent": true, + "provider_objects_retained": false + }, + "schema": "gda.real_feature_ingestion_observation.v1", + "source_projection": { + "bounds": [ + 106.37987914500007, + 29.558008447000077, + 106.59532712300006, + 29.87727198500005 + ], + "feature_count": 20, + "geometry_types": [ + "MultiPolygon", + "Polygon" + ], + "geometry_z_count": 20, + "non_empty_geometry_count": 20, + "payload_sha256": "f60e33c9ef890eb20c87e6cde82e658aaa3b20151e528ffbb293b9b64fabbbe1", + "payload_size_bytes": 430066, + "row_set_sha256": "c26ff708f4b6be082327dff63a6a8659420dbc4cab37dea1cac7b40f147512df", + "row_sha256": [ + "0527a75a86c70503302f673535003792e135c0097fcb3a44ec3b081ab74e4a76", + "08113e2f628a656b9e3a23ada3b94e0c7568ec7585ae9f25efe7702662d1f2fd", + "176093ff06f206a65fa98bcc0cb0851cb66b91b8f223d28c1b60317298ca9190", + "2305d14edf0ba61478a7e8d0511d904e36574dd691c7e580026416ed891cc18a", + "307531c08ac817f34fd4f40b2f0e9afc96758e67a4a9af2c199a9966242ffc88", + "492fee3e36f8a4ba7919ffcd6b9c4d1977baf335670a7d81159e55ee89d38506", + "4939fa3bb87d941c8a5a3e328f45bb0077982fa7d172968ed9afc76495a0a8bd", + "52de730203b330e1fa66790c2f6670d3248c0cb2620788c045e26aba3f4263a0", + "62b781771b8aa704b34fe556ace2afa47e843611fbf82c4f487ac5323a956952", + "6e96076e7ee2cfbc37b5bd093eedcd1bc9a03a4618fb30f5a7d63581aad77ca9", + "7c5d3112549cfc3b8c95cad896179060d2c9b9c5e56edd9646026ba2c9ffc039", + "8d346f09d4b606cd7ade5b196d10a3a5f54b3dff40205f1e21bea4aae10df5da", + "947b9485ed7d084e02ef780e82fd6a4fb1c37c601fe19a4be37962cd5526979b", + "a55e54b78a5b64ee6e629586968f7b304521b28be94b55b9c7ebe8670e0b83db", + "c284f86d861e265b918fd14a3f5653133bbdcd01f7f52d1cc3cde509044106ff", + "c55ed9b57879c11ea1f606e3318147893d54a4f02e9e96d4948e92cfc0880eb7", + "c84e602b9cb743f5f40b4e7c93ee4322e62c23edecf90d4b6d14c441e7c5113f", + "ce29800e1a902c1c1435277f328b599a05a7d431543cf9994ef2528f7913e34f", + "e12c4abaa5df83bdccfc33852a2f6e2e2483aa3c0ce2e77e0f470e9e76113968", + "e271fe60f1ecf40813c21b727a88b9f95b2a07c5e8df797d9370414e1a161d23" + ], + "schema": "gda.real_feature_row_set.v1", + "source_payload_recorded": false, + "srid": 4490, + "unique_identifier_count": 20, + "valid_geometry_count": 20 + }, + "spark": { + "failure_diagnostic": [], + "job": { + "completion_time": "2026-07-31T01:15:32Z", + "failed": 0, + "name": "real-feature-ingestion-probe", + "succeeded": 1, + "uid": "4d524055-72e7-4614-86d2-b36c55707b9a" + }, + "log_recorded": false, + "log_sha256": "e1be1925e6bd4985d6161cbe9812cb99f9df6a6e647fbcd001da3dd0a92ac6c5", + "pod": { + "image": "docker.io/gisdataagent/mmfe-spark-runtime:local", + "image_id": "sha256:4a4522bfd4e6d1c6c90a244d0145841fbfbbf21ed16ee29ca8b681b5cec60058", + "name": "real-feature-ingestion-probe-8pvrn", + "node_name": "desktop-worker", + "persistent_volume_claims": [], + "phase": "Succeeded", + "service_account": "spark-object-store-probe", + "service_account_automount_disabled": true, + "uid": "ef842b60-eeab-43b5-9b80-343a93a35d45" + }, + "result": { + "authorization_sha256": "7eb8bfae2306221e1f816d139a5347df8d3b7107058417cebb10e038f2175d9c", + "first_execution": { + "data_files": [ + { + "file_path": "s3://gda-metadata-warehouse/warehouse/cultural_heritage/cultural_districts/data/00000-8-11851edc-05b2-4ce0-9a1c-17b7ef344b4d-0-00001.parquet", + "record_count": 20 + } + ], + "mutation_count": 1, + "row_sha256": [ + "0527a75a86c70503302f673535003792e135c0097fcb3a44ec3b081ab74e4a76", + "08113e2f628a656b9e3a23ada3b94e0c7568ec7585ae9f25efe7702662d1f2fd", + "176093ff06f206a65fa98bcc0cb0851cb66b91b8f223d28c1b60317298ca9190", + "2305d14edf0ba61478a7e8d0511d904e36574dd691c7e580026416ed891cc18a", + "307531c08ac817f34fd4f40b2f0e9afc96758e67a4a9af2c199a9966242ffc88", + "492fee3e36f8a4ba7919ffcd6b9c4d1977baf335670a7d81159e55ee89d38506", + "4939fa3bb87d941c8a5a3e328f45bb0077982fa7d172968ed9afc76495a0a8bd", + "52de730203b330e1fa66790c2f6670d3248c0cb2620788c045e26aba3f4263a0", + "62b781771b8aa704b34fe556ace2afa47e843611fbf82c4f487ac5323a956952", + "6e96076e7ee2cfbc37b5bd093eedcd1bc9a03a4618fb30f5a7d63581aad77ca9", + "7c5d3112549cfc3b8c95cad896179060d2c9b9c5e56edd9646026ba2c9ffc039", + "8d346f09d4b606cd7ade5b196d10a3a5f54b3dff40205f1e21bea4aae10df5da", + "947b9485ed7d084e02ef780e82fd6a4fb1c37c601fe19a4be37962cd5526979b", + "a55e54b78a5b64ee6e629586968f7b304521b28be94b55b9c7ebe8670e0b83db", + "c284f86d861e265b918fd14a3f5653133bbdcd01f7f52d1cc3cde509044106ff", + "c55ed9b57879c11ea1f606e3318147893d54a4f02e9e96d4948e92cfc0880eb7", + "c84e602b9cb743f5f40b4e7c93ee4322e62c23edecf90d4b6d14c441e7c5113f", + "ce29800e1a902c1c1435277f328b599a05a7d431543cf9994ef2528f7913e34f", + "e12c4abaa5df83bdccfc33852a2f6e2e2483aa3c0ce2e77e0f470e9e76113968", + "e271fe60f1ecf40813c21b727a88b9f95b2a07c5e8df797d9370414e1a161d23" + ], + "snapshots": [ + { + "operation": "append", + "parent_id": null, + "snapshot_id": 6084664108418947049 + } + ], + "status": "appended" + }, + "iceberg_runtime": "1.6.1", + "immediate_replay": { + "data_files": [ + { + "file_path": "s3://gda-metadata-warehouse/warehouse/cultural_heritage/cultural_districts/data/00000-8-11851edc-05b2-4ce0-9a1c-17b7ef344b4d-0-00001.parquet", + "record_count": 20 + } + ], + "mutation_count": 0, + "row_sha256": [ + "0527a75a86c70503302f673535003792e135c0097fcb3a44ec3b081ab74e4a76", + "08113e2f628a656b9e3a23ada3b94e0c7568ec7585ae9f25efe7702662d1f2fd", + "176093ff06f206a65fa98bcc0cb0851cb66b91b8f223d28c1b60317298ca9190", + "2305d14edf0ba61478a7e8d0511d904e36574dd691c7e580026416ed891cc18a", + "307531c08ac817f34fd4f40b2f0e9afc96758e67a4a9af2c199a9966242ffc88", + "492fee3e36f8a4ba7919ffcd6b9c4d1977baf335670a7d81159e55ee89d38506", + "4939fa3bb87d941c8a5a3e328f45bb0077982fa7d172968ed9afc76495a0a8bd", + "52de730203b330e1fa66790c2f6670d3248c0cb2620788c045e26aba3f4263a0", + "62b781771b8aa704b34fe556ace2afa47e843611fbf82c4f487ac5323a956952", + "6e96076e7ee2cfbc37b5bd093eedcd1bc9a03a4618fb30f5a7d63581aad77ca9", + "7c5d3112549cfc3b8c95cad896179060d2c9b9c5e56edd9646026ba2c9ffc039", + "8d346f09d4b606cd7ade5b196d10a3a5f54b3dff40205f1e21bea4aae10df5da", + "947b9485ed7d084e02ef780e82fd6a4fb1c37c601fe19a4be37962cd5526979b", + "a55e54b78a5b64ee6e629586968f7b304521b28be94b55b9c7ebe8670e0b83db", + "c284f86d861e265b918fd14a3f5653133bbdcd01f7f52d1cc3cde509044106ff", + "c55ed9b57879c11ea1f606e3318147893d54a4f02e9e96d4948e92cfc0880eb7", + "c84e602b9cb743f5f40b4e7c93ee4322e62c23edecf90d4b6d14c441e7c5113f", + "ce29800e1a902c1c1435277f328b599a05a7d431543cf9994ef2528f7913e34f", + "e12c4abaa5df83bdccfc33852a2f6e2e2483aa3c0ce2e77e0f470e9e76113968", + "e271fe60f1ecf40813c21b727a88b9f95b2a07c5e8df797d9370414e1a161d23" + ], + "snapshots": [ + { + "operation": "append", + "parent_id": null, + "snapshot_id": 6084664108418947049 + } + ], + "status": "no_op" + }, + "material_recorded": false, + "output_content_sha256": "bdc06792e8b935176ee6df6f6f6d4be1535622d54d9b994a778cabfe5a574618", + "output_resource_version_id": "a6000000-0000-4000-8000-000000000002", + "plan_sha256": "6282e636cc9399e5f8ddc41defb2123a2a31d1d2ae925e3b3660c48198dfbf84", + "quality": { + "bbox_match_count": 20, + "feature_count": 20, + "positive_area_count": 20, + "srid_match_count": 20, + "unique_bsm_count": 20, + "valid_geometry_count": 20 + }, + "row_set_sha256": "c26ff708f4b6be082327dff63a6a8659420dbc4cab37dea1cac7b40f147512df", + "schema": "gda.real_feature_ingestion_probe_result.v1", + "sedona_version": "1.9.0", + "source_content_sha256": "fd474fd65c8e4a71da241eb3fd07748ca3b972fbd2d3c32833376dbe71104007", + "source_payload_recorded": false, + "source_resource_version_id": "a6000000-0000-4000-8000-000000000001", + "spark_version": "3.5.0", + "table": "rest.cultural_heritage.cultural_districts", + "table_columns": [ + "BSM", + "geometry", + "srid", + "min_x", + "min_y", + "max_x", + "max_y", + "row_sha256" + ] + }, + "result_line_count": 1, + "terminal_condition": "Complete", + "wait_completed": true + }, + "table_create": { + "mutation_count": 1, + "mutations": [ + "gravitino.table.create" + ], + "source_binding_verified": true, + "status": "created", + "table_projection": { + "columns": [ + { + "name": "BSM", + "nullable": false, + "type": "string" + }, + { + "name": "geometry", + "nullable": false, + "type": "binary" + }, + { + "name": "srid", + "nullable": false, + "type": "integer" + }, + { + "name": "min_x", + "nullable": false, + "type": "double" + }, + { + "name": "min_y", + "nullable": false, + "type": "double" + }, + { + "name": "max_x", + "nullable": false, + "type": "double" + }, + { + "name": "max_y", + "nullable": false, + "type": "double" + }, + { + "name": "row_sha256", + "nullable": false, + "type": "string" + } + ], + "content_sha256": "bdc06792e8b935176ee6df6f6f6d4be1535622d54d9b994a778cabfe5a574618", + "name": "cultural_districts", + "provider_revision": "m3-22-real-feature-ingestion-v1", + "resource_urn": "gda://metadata-authorization-local/data_product/chongqing-cultural-districts-iceberg", + "resource_version_id": "a6000000-0000-4000-8000-000000000002" + }, + "table_projection_sha256": "f30feb94a5a8280597f331a7f965762bfabb9b82397af49dda05b61ce00bbb1e" + } + }, + "oidc_verified": false, + "output_resource_version_id": "a6000000-0000-4000-8000-000000000002", + "path_free_lineage_candidate_verified": true, + "platform_run_succeeded": false, + "predecessor_history_changed": false, + "predecessor_promotion_candidate_sha256": "63812c311b3f239bc6a944748c4ff384250eb9c9ed9009d3384fc699f1d3eaa9", + "production_ingestion_verified": false, + "production_object_store_verified": false, + "production_ready": false, + "protected_workload_identity_verified": false, + "real_dataset_resource_version_bound": true, + "schema": "gda.real_feature_ingestion_evidence.v1", + "sedona_spatial_quality_verified": true, + "source_absolute_path_committed": false, + "source_dataset_committed": false, + "source_feature_payload_committed": false, + "source_projection": { + "bounds": [ + 106.37987914500007, + 29.558008447000077, + 106.59532712300006, + 29.87727198500005 + ], + "feature_count": 20, + "geometry_types": [ + "MultiPolygon", + "Polygon" + ], + "geometry_z_count": 20, + "non_empty_geometry_count": 20, + "payload_sha256": "f60e33c9ef890eb20c87e6cde82e658aaa3b20151e528ffbb293b9b64fabbbe1", + "payload_size_bytes": 430066, + "row_set_sha256": "c26ff708f4b6be082327dff63a6a8659420dbc4cab37dea1cac7b40f147512df", + "row_sha256": [ + "0527a75a86c70503302f673535003792e135c0097fcb3a44ec3b081ab74e4a76", + "08113e2f628a656b9e3a23ada3b94e0c7568ec7585ae9f25efe7702662d1f2fd", + "176093ff06f206a65fa98bcc0cb0851cb66b91b8f223d28c1b60317298ca9190", + "2305d14edf0ba61478a7e8d0511d904e36574dd691c7e580026416ed891cc18a", + "307531c08ac817f34fd4f40b2f0e9afc96758e67a4a9af2c199a9966242ffc88", + "492fee3e36f8a4ba7919ffcd6b9c4d1977baf335670a7d81159e55ee89d38506", + "4939fa3bb87d941c8a5a3e328f45bb0077982fa7d172968ed9afc76495a0a8bd", + "52de730203b330e1fa66790c2f6670d3248c0cb2620788c045e26aba3f4263a0", + "62b781771b8aa704b34fe556ace2afa47e843611fbf82c4f487ac5323a956952", + "6e96076e7ee2cfbc37b5bd093eedcd1bc9a03a4618fb30f5a7d63581aad77ca9", + "7c5d3112549cfc3b8c95cad896179060d2c9b9c5e56edd9646026ba2c9ffc039", + "8d346f09d4b606cd7ade5b196d10a3a5f54b3dff40205f1e21bea4aae10df5da", + "947b9485ed7d084e02ef780e82fd6a4fb1c37c601fe19a4be37962cd5526979b", + "a55e54b78a5b64ee6e629586968f7b304521b28be94b55b9c7ebe8670e0b83db", + "c284f86d861e265b918fd14a3f5653133bbdcd01f7f52d1cc3cde509044106ff", + "c55ed9b57879c11ea1f606e3318147893d54a4f02e9e96d4948e92cfc0880eb7", + "c84e602b9cb743f5f40b4e7c93ee4322e62c23edecf90d4b6d14c441e7c5113f", + "ce29800e1a902c1c1435277f328b599a05a7d431543cf9994ef2528f7913e34f", + "e12c4abaa5df83bdccfc33852a2f6e2e2483aa3c0ce2e77e0f470e9e76113968", + "e271fe60f1ecf40813c21b727a88b9f95b2a07c5e8df797d9370414e1a161d23" + ], + "schema": "gda.real_feature_row_set.v1", + "source_payload_recorded": false, + "srid": 4490, + "unique_identifier_count": 20, + "valid_geometry_count": 20 + }, + "source_resource_version_id": "a6000000-0000-4000-8000-000000000001", + "spark_conformance_verified": false, + "status": "local_real_feature_ingestion_verified", + "tls_verified": false +} diff --git a/docs/roadmap-ar0-platform-truth-2026-07-24.md b/docs/roadmap-ar0-platform-truth-2026-07-24.md index e3e24ba8..aacbfce2 100644 --- a/docs/roadmap-ar0-platform-truth-2026-07-24.md +++ b/docs/roadmap-ar0-platform-truth-2026-07-24.md @@ -199,7 +199,7 @@ Temporal 继续保持目标组件状态，不在这一包并行接入。OpenMeta 当前完成仅指本地合同、授权 evidence、outbox/callback 代码、数据库成功终局门、托管 worker 代码、默认关闭的部署模板及离线 activation/release preflight、candidate/registry/provenance/artifact-release/live observation evidence gate、合成 golden slice、定向测试、真实 PostgreSQL 16 事务边界和 canonical mainline 治理。`candidate_validated`、`registry_subject_bound`、本地合成 `provenance_verified`、`ready_for_activation`、`ready_for_staging_apply`、`verified_for_staging_apply` 和本地 live collection 都不等于真实镜像已 attested 或 staging 已部署；真实 IAM/OIDC 与 service token 生命周期、首次 GHCR publish/verify、真实 provenance artifact verify、registry-backed live staging revision、worker/callback 扩容运行、golden slice staging 运行链、受保护 release/live evidence provenance、独立 DolphinScheduler metadata PostgreSQL 和真实数据终局证据仍属于 4.7 后续切片。 -### 4.8 Metadata Fabric Bridge M1 + M2 + M3-21（本地 JDBC/S3 runtime-bound promotion 已验证，生产验证待执行） +### 4.8 Metadata Fabric Bridge M1 + M2 + M3-22（本地真实要素 JDBC/S3 ingestion 已验证，权威晋级与生产验证待执行） 第八块回到 AR-1 的 metadata control plane，以 [ADR-036](architecture-decisions/adr-036-read-only-metadata-fabric-bridge-contract.md) 固定 OpenMetadata + Gravitino + GDA Control Ledger 的首条 table slice： @@ -241,10 +241,11 @@ Temporal 继续保持目标组件状态，不在这一包并行接入。OpenMeta 36. [ADR-065](architecture-decisions/adr-065-local-active-metadata-binding-reconciliation.md) 在提交 M3-18 binding 前先验证 retained OpenMetadata UUID/FQN/version/content/governance/snapshot 完全一致。Gravitino `memory` catalog 重启后出现 connector 空状态与 provider entity index 残留的分裂；M3-19 只在专用 catalog 配置精确且可见 schema inventory 为空时执行 provider-native reset，并以 4 个 `gravitino.*` mutations 重建 catalog/schema/table，OpenMetadata 零写入。即时 replay 为 `no_op/0 mutations` 且 binding SHA 仍为 `7de24cee9dd50dfeefcc886cf43024f4d92b7650767d71d064fdce19ffccb16b`。PlatformGateway 首次 binding commit `created=true`、重放 `created=false`、仅 1 行，FORCE RLS、跨租户隔离、append-only 和 direct UPDATE/DELETE 拒绝均通过；Run 保持 `reconciling`，所有临时资源清理。contract fingerprint 为 `012a7c86ba9fe53217e721ff7286b8f2a246b9394efd2999abbcd025e13ac7f5`，evidence fingerprint 为 `e6d0e3ac4e052029dad0c18d0804626a8af61554a54081c37d8cc9a80c55cd33`。`durable_catalog_verified=false`，该结论不证明生产 identity/catalog/executor/binding deployment、production ingestion 或 `production_ready`。 37. [ADR-066](architecture-decisions/adr-066-runtime-bound-durable-active-metadata-promotion.md) 保持 M3-19 binding schema/ledger/evidence 不变，将同一重庆 ResourceVersion 投影到隔离 JDBC metadata + warehouse PVC 的 Gravitino target，并把 logical provider ref 与 cluster/namespace/Service/StatefulSet/PVC/image identity 组合为独立 promotion candidate。受限 Basic principal 首次只执行 1 个 `gravitino.table.create`；即时 replay 与 PostgreSQL -> Gravitino 有序 restart 后的第一次 replay 都为 `no_op/0 mutations`，两次 Pod UID 变化而稳定 runtime/PVC identity 与 table projection 不变。logical binding SHA 为 `8c312db37bfe92e034bcdcb7a3c35847c81e862c74a3437970def1007af42750`，runtime binding SHA 为 `a78975311fc34abd76fa41dea581594806b3d18ed364ba518cfc44c4204822f7`，promotion candidate SHA 为 `bb6672cb7f98fa53305e17bbca2cb5b3756d4a335a94d79114fb4184273871d1`，contract/evidence SHA 分别为 `307f2d4390028589c0f38be859c53826bd149d7f2a133b14488230d4f5ff6eb8` / `53773e9417668e03ad3ab2b5c3cdbd627fb3bc397d63c5860755ec5318eebe8b`。candidate 未写 GDA Control，namespace/PVC 已清理；`durable_catalog_verified=false`、`production_object_store_verified=false`、`production_ready=false`。 38. [ADR-067](architecture-decisions/adr-067-object-store-runtime-bound-active-metadata-promotion.md) 以 M3-20 promotion candidate 为不可变 predecessor，在 M3-10 跨节点 MinIO runtime 中创建独立 `gda_chongqing_m3_21` JDBC/S3 target。runtime binding 同时包含 Gravitino/MinIO Service、PostgreSQL/Gravitino/MinIO StatefulSet、PostgreSQL/MinIO PVC、镜像、节点分离与 S3 warehouse/endpoint/bucket，且 Gravitino 无 warehouse PVC。受限 principal 首次仅 1 个 table create，即时及 PostgreSQL -> Gravitino restart 后首个 replay 均为 `no_op/0`；S3 直读在精确 prefix 下只见 1 个 Iceberg metadata JSON，无 data/manifest，key/ETag/body SHA/表 schema 重启前后不变。predecessor/logical/runtime/promotion SHA 分别为 `bb6672cb7f98fa53305e17bbca2cb5b3756d4a335a94d79114fb4184273871d1`、`614ce5e4c45dba1437dc888cbd79b2d58954184113a62c20170ab84b5570d9e1`、`dd63917b6354a2e92853763ddc3e3a981cb40717f84c0f819b1a4e6844ae100b`、`63812c311b3f239bc6a944748c4ff384250eb9c9ed9009d3384fc699f1d3eaa9`；contract/evidence SHA 为 `b1a2db34a70eaa7dd55da1d6c85da9f420c755c71868aafe7972e3794034a6cc` / `d73754c53cf16d888aa345baa5d079cc7fd98d8b84db747f52188c1a69bf1628`。candidate 未落账、未 ingest feature rows，namespace/PVC/port-forward 已清理；生产对象存储与 readiness 仍为 `false`。 +39. [ADR-068](architecture-decisions/adr-068-local-authorized-real-feature-iceberg-ingestion.md) 复用同一重庆 20-feature EPSG:4490 bundle 和 M3-21 predecessor，在独立 `gda_chongqing_m3_22` target 中由受限 Gravitino principal 创建八列表，再由一个内容与授权指纹绑定的 Spark `3.5.0` + Sedona `1.9.0` Job 写入。六项空间质量计数均为 20；首次执行 `appended/1`，产生 1 个 snapshot 和 1 个 20-row Parquet，即时 replay 为 `no_op/0` 且 row/snapshot/file readback 不变。S3 直读为 1 data + 2 metadata + 2 manifest，并构造独立 output ResourceVersion、Artifact、passed QualityResult 和 LineageEvent candidates。row-set/output/contract/evidence SHA 分别为 `c26ff708f4b6be082327dff63a6a8659420dbc4cab37dea1cac7b40f147512df`、`bdc06792e8b935176ee6df6f6f6d4be1535622d54d9b994a778cabfe5a574618`、`af211f2d2f4830decb9ffe369cd9e7ec2c9349c2e2c8bd789347a6fdc288e1dc`、`42abd82613eaf28cb53c64280258bc75dba6cf841f9a513a4c801a9f798b9899`。源路径、BSM、WKB 和 credential 不入 evidence，namespace/PVC/port-forward 已清理；candidates 未落 GDA Control，Run 未终局，生产 ingestion/readiness 仍为 `false`。 -M3-21 只证明本地同主机双节点上的 JDBC/S3 provider identity、空表 Iceberg metadata 与 restart continuity；它不替代真实 feature-row ingestion、生产对象存储 attestation、独立 failure domain、KMS/TLS/workload identity、备份/PITR、tenant isolation 或权威 promotion ledger。 +M3-22 已证明本地同主机双节点上一个受授权真实 feature slice 的空间质量、Iceberg 单次 append、即时 no-op replay、S3 直读和 path-free output candidates；它不替代权威 GDA Control promotion、Run 成功终局、staging/大规模 ingestion、生产对象存储 attestation、独立 failure domain、KMS/TLS/workload identity、备份/PITR 或 tenant isolation。 -此处 M1 只证明静态合同和只读 HTTP 边界；M2a 只证明本地 live foundation 与 PVC 重挂载连续性；M2b-1/M2b-2 分别限定在同集群新 PVC 和同集群隔离 repository；M2b-3 的 `local_cross_cluster_recovery_verified=true` 只限定在 `local_same_host_distinct_kubernetes_clusters_external_s3_repository`；M2c-1/M2c-2/M2c-3 分别限定本地 provider metrics、临时双周期 OTel 和单 job scrape recovery；M2c-4/M2d-2 只证明 production observability/NetworkPolicy profile 与 attestation 合同可校验；M2d-1 只证明本地两节点 kindnet 的隔离合成流量；M3-1 的 terminal evidence 与 M3-2 的 PolicyDecision/Approval 仍是 deterministic local fixtures。M3-2 只把 projection 写入本地 provider 并证明 retained target 的单次零写入 replay；M3-3 只把该本地 evidence 对应的 binding 写入临时 GDA Control 账本；M3-4 只向无认证 loopback receiver 发送精确 candidate 并验证 503 后幂等恢复；M3-5 只证明 OpenMetadata 在 provider 强制默认 role 之上的项目新增 grant 限定为 `table/Create`，以及本地 JWT 轮换/吊销和越权拒绝；M3-6 只证明隔离 Gravitino Basic IdP 的 bounded table-create、catalog-create 拒绝、登录轮换/吊销和完整清理；M3-7 只证明 pending production identity profile、profile-bound attestation 和派生 claim 的 fail-closed 合同可校验，没有部署或证明真实身份路径；M3-8 只证明同一 Docker Desktop 集群内 Basic 用户、JDBC metadata 与 file warehouse PVC 在受控 Pod restart 后连续；M3-9 只证明同节点共享 RWO PVC 的 Spark interoperability；M3-10 移除了该共享 PVC，并证明同一 Docker Desktop 主机/集群内 Spark 与 MinIO 的跨节点 S3-compatible 互操作，但不证明生产云对象存储、独立 failure domain、持久 identity binding、Flink 或完整 engine conformance；M3-11 只冻结 provider-neutral production object-store profile、精确 attestation binding 与 fail-closed claims，没有选择 provider、部署 bucket/KMS/policy 或提交真实 attestation；M3-12 只证明同一本地路径的 pre-forward commit failure 不改变可见 table state，随后一次显式重试产生一个新 snapshot/row，且无孤儿 data file；M3-13 只证明单次本地 append 在 provider 200 响应丢失并映射为 commit-state-unknown 后，可以由即时 table readback 判定 committed 且不重提，不覆盖持久 controller、进程崩溃、并发写或任意 mutation；M3-14 只证明 ResourceVersion 注册与 Active Metadata 事件在本地 PostgreSQL 同事务创建，并验证租户/workload scoped claim/retry/complete；M3-15 只证明默认零副本 managed consumer 的代码/部署边界，以及本地 PostgreSQL 中 inert activation request 与 event completion 的原子性；M3-16 只证明本地真实数据 content fingerprint、证据绑定授权与 pending command 的 PostgreSQL 原子性；M3-17 只证明本地 standalone 中既有 consumer/adapter 的真实 submission、精确 correlation read-back 和 provider success observation；M3-18 只证明同一 Docker Desktop 主机上 scheduler 通过 ephemeral HTTP executor 触发 bootstrap-admin/unauthenticated providers 的一次创建和同进程零写 replay；M3-19 只证明同一主机上 exact OpenMetadata + absent Gravitino 的受限修复、即时 no-op replay 和临时 PostgreSQL binding commit，且 `memory` catalog reset 明确不等于 durable catalog recovery。生产持久 binding deployment、ResourceVersion 和 legacy authority 都未切换；生产对象存储、双 provider/生产最小权限、protected workload identity、OIDC、TLS、生产持久 catalog、tenant isolation、真实 receiver/alert/SLO、受保护 provider policy、生产故障注入、source-loss recovery、cancel/reconcile/lineage、完整 Spark/Flink conformance、生产 ingest、四项 production gate 和 `production_ready` 仍为 `false`。 +此处 M1 只证明静态合同和只读 HTTP 边界；M2a 只证明本地 live foundation 与 PVC 重挂载连续性；M2b-1/M2b-2 分别限定在同集群新 PVC 和同集群隔离 repository；M2b-3 的 `local_cross_cluster_recovery_verified=true` 只限定在 `local_same_host_distinct_kubernetes_clusters_external_s3_repository`；M2c-1/M2c-2/M2c-3 分别限定本地 provider metrics、临时双周期 OTel 和单 job scrape recovery；M2c-4/M2d-2 只证明 production observability/NetworkPolicy profile 与 attestation 合同可校验；M2d-1 只证明本地两节点 kindnet 的隔离合成流量；M3-1 的 terminal evidence 与 M3-2 的 PolicyDecision/Approval 仍是 deterministic local fixtures。M3-2 只把 projection 写入本地 provider 并证明 retained target 的单次零写入 replay；M3-3 只把该本地 evidence 对应的 binding 写入临时 GDA Control 账本；M3-4 只向无认证 loopback receiver 发送精确 candidate 并验证 503 后幂等恢复；M3-5 只证明 OpenMetadata 在 provider 强制默认 role 之上的项目新增 grant 限定为 `table/Create`，以及本地 JWT 轮换/吊销和越权拒绝；M3-6 只证明隔离 Gravitino Basic IdP 的 bounded table-create、catalog-create 拒绝、登录轮换/吊销和完整清理；M3-7 只证明 pending production identity profile、profile-bound attestation 和派生 claim 的 fail-closed 合同可校验，没有部署或证明真实身份路径；M3-8 只证明同一 Docker Desktop 集群内 Basic 用户、JDBC metadata 与 file warehouse PVC 在受控 Pod restart 后连续；M3-9 只证明同节点共享 RWO PVC 的 Spark interoperability；M3-10 移除了该共享 PVC，并证明同一 Docker Desktop 主机/集群内 Spark 与 MinIO 的跨节点 S3-compatible 互操作，但不证明生产云对象存储、独立 failure domain、持久 identity binding、Flink 或完整 engine conformance；M3-11 只冻结 provider-neutral production object-store profile、精确 attestation binding 与 fail-closed claims，没有选择 provider、部署 bucket/KMS/policy 或提交真实 attestation；M3-12 只证明同一本地路径的 pre-forward commit failure 不改变可见 table state，随后一次显式重试产生一个新 snapshot/row，且无孤儿 data file；M3-13 只证明单次本地 append 在 provider 200 响应丢失并映射为 commit-state-unknown 后，可以由即时 table readback 判定 committed 且不重提，不覆盖持久 controller、进程崩溃、并发写或任意 mutation；M3-14 只证明 ResourceVersion 注册与 Active Metadata 事件在本地 PostgreSQL 同事务创建，并验证租户/workload scoped claim/retry/complete；M3-15 只证明默认零副本 managed consumer 的代码/部署边界，以及本地 PostgreSQL 中 inert activation request 与 event completion 的原子性；M3-16 只证明本地真实数据 content fingerprint、证据绑定授权与 pending command 的 PostgreSQL 原子性；M3-17 只证明本地 standalone 中既有 consumer/adapter 的真实 submission、精确 correlation read-back 和 provider success observation；M3-18 只证明同一 Docker Desktop 主机上 scheduler 通过 ephemeral HTTP executor 触发 bootstrap-admin/unauthenticated providers 的一次创建和同进程零写 replay；M3-19 只证明同一主机上 exact OpenMetadata + absent Gravitino 的受限修复、即时 no-op replay 和临时 PostgreSQL binding commit，且 `memory` catalog reset 明确不等于 durable catalog recovery；M3-22 只证明短生命周期本地 runtime 中一份 20-row 真实 slice 的授权写入、质量 readback 和 path-free candidates，未向 GDA Control 晋级或形成 Run 成功终局。生产持久 binding deployment、ResourceVersion 和 legacy authority 都未切换；生产对象存储、双 provider/生产最小权限、protected workload identity、OIDC、TLS、生产持久 catalog、tenant isolation、真实 receiver/alert/SLO、受保护 provider policy、生产故障注入、source-loss recovery、cancel/reconcile/lineage、完整 Spark/Flink conformance、生产 ingest、四项 production gate 和 `production_ready` 仍为 `false`。 ## 5. 重新评估条件 diff --git a/docs/system-of-record-matrix-2026-07-24.md b/docs/system-of-record-matrix-2026-07-24.md index f3c1e7b6..79282a5b 100644 --- a/docs/system-of-record-matrix-2026-07-24.md +++ b/docs/system-of-record-matrix-2026-07-24.md @@ -1,10 +1,10 @@ # GIS Data Agent System-of-Record 矩阵 -日期：2026-07-30 +日期：2026-07-31 -阶段：AR-0 `in_progress`；AR-1 gateway、成功终局 evidence gate、DolphinScheduler adapter sandbox POC、Metadata Fabric M1/M2、M2c-4/M2d-2 production readiness contracts、M3-1 至 M3-19 local Active Metadata binding reconciliation 已验证，生产 provider ingestion、生产观测、生产 policy/tenant isolation、生产 identity/object-store attestation、生产 consumer/scheduler/executor 和生产切换仍 `in_progress` +阶段：AR-0 `in_progress`；AR-1 gateway、成功终局 evidence gate、DolphinScheduler adapter sandbox POC、Metadata Fabric M1/M2、M2c-4/M2d-2 production readiness contracts、M3-1 至 M3-22 local authorized real-feature ingestion 已验证，权威 output promotion、生产观测、生产 policy/tenant isolation、生产 identity/object-store attestation、生产 consumer/scheduler/executor 和生产切换仍 `in_progress` -适用分支：`feat/ar1-metadata-fabric-active-metadata-binding-reconciliation` +适用分支：`feat/ar1-metadata-fabric-real-feature-ingestion` ## 判定规则 @@ -65,6 +65,7 @@ 20. M3-19 只允许在 retained OpenMetadata UUID/FQN/version/content/governance/snapshot 完全匹配时修复缺失 Gravitino target；专用 `memory` catalog 只有配置精确且可见 schema inventory 为空才可 provider-native reset。修复限于 4 个 `gravitino.*` mutations，OpenMetadata 零写入，即时 replay 为 `no_op/0 mutations`，binding 通过临时 PostgreSQL PlatformGateway 幂等追加且 Run 留在 `reconciling`。这不证明 durable catalog、protected identity、生产 executor/scheduler/provider、生产 binding deployment/ingestion 或 terminal success。 21. M3-20 不修改 M3-19 binding schema、ledger 或 evidence，而是为同一重庆 ResourceVersion 新建 runtime-bound durable promotion candidate。受限 Basic principal 在隔离 JDBC metadata + warehouse PVC target 中只创建一次表；即时 replay 与 PostgreSQL/Gravitino restart 后第一次 replay 均为 `no_op/0 mutations`。cluster/namespace/Service/StatefulSet/PVC/image identity 被绑定且重启前后稳定，Pod UID 必须变化；candidate 未写 GDA Control，namespace/PVC 已清理。这只证明本地 restart continuity，不证明生产 durable catalog/object store、protected identity、OIDC/TLS、生产 ingestion 或 readiness。 22. M3-21 不修改 M3-20/M3-19 历史，而是以 M3-20 candidate 为 predecessor，将同一重庆 ResourceVersion 投影到 JDBC catalog + 跨节点 MinIO warehouse。稳定 binding 包含双 Service、三个 StatefulSet、PostgreSQL/MinIO PVC、镜像、节点和 S3 配置，Gravitino 无 warehouse PVC。首次 apply 为 1 个 table create，即时及有序重启后首个 replay 均为 `no_op/0`；直接 S3 metadata key/ETag/body SHA/表 schema 不变。该表没有 source feature rows，candidate 未落账，所有临时资源已清理。这不证明生产对象存储、durable catalog、protected identity、TLS/OIDC、生产 ingestion 或 readiness。 +23. M3-22 以 M3-21 candidate 为 predecessor，将同一重庆 bundle 的 20 个真实 EPSG:4490 feature rows 规范化为八列、由精确 PolicyDecision/Approval 授权 Spark/Sedona 写入 JDBC/S3 Iceberg。六项质量计数均为 20，首次执行 `appended/1` 且只有 1 个 snapshot/Parquet，即时 replay 为 `no_op/0`；S3 直读为 1 data + 2 metadata + 2 manifest。输出 ResourceVersion、Artifact、passed QualityResult 与 LineageEvent 只是 path-free candidates，未写 GDA Control，Run 未成功终局；namespace/PV/port-forward 已清理。这不证明生产对象存储、protected identity、完整 engine conformance、生产 ingestion 或 readiness。 ## 已建立的 AR-0/AR-1 entry 证据 @@ -112,7 +113,8 @@ ## 下一验收证据 -- M3-21 的本地 MinIO 空表 metadata promotion 不计入生产对象存储或 ingestion 退出门；下一步需要受授权 Spark/Sedona 写入真实重庆 feature slice，并继续保持生产 provider attestation 与 ledger promotion 独立验收； +- M3-21 的本地 MinIO 空表 metadata promotion 不计入生产对象存储或 ingestion 退出门；其真实重庆 feature slice 后续由 M3-22 独立验收，历史 candidate 与 evidence 保持不变； +- M3-22 的本地真实 feature slice 已通过受授权 Spark/Sedona、单 snapshot/no-op replay 与直接 S3 readback，但不计入生产 ingestion 退出门；下一步是将 output/quality/lineage candidates 原子晋级到 GDA Control，并继续把 Run 终局、生产 provider attestation 与 staging-scale ingestion 独立验收； - 完成首次 application subject publish 与 protected verifier run；当前 mainline、archive refs、ruleset、required reviewer、禁止 bypass 和 environment enable variable 已配置并复核； - 真实 provenance artifact verify、受保护 overlay 的 `verified_for_staging_apply` release report，以及 staging/production 的 schema、config/runtime snapshot、registry/live DeploymentRevision 绑定、release/live artifact attestation 和环境 compare 报告； - staging 的 migration role、应用 login membership、连接池 role/tenant 复位、双租户 API 和 success finalization 运行产物； diff --git a/k8s/metadata-fabric-real-feature-ingestion/spark-job.yaml b/k8s/metadata-fabric-real-feature-ingestion/spark-job.yaml new file mode 100644 index 00000000..5e76e214 --- /dev/null +++ b/k8s/metadata-fabric-real-feature-ingestion/spark-job.yaml @@ -0,0 +1,300 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: real-feature-ingestion-probe + namespace: gda-metadata-spark-object-store +data: + probe.py: | + import json + import os + + from pyspark.sql import functions as F + from pyspark.sql import types as T + from sedona.spark import SedonaContext + + + TABLE = "rest.cultural_heritage.cultural_districts" + INPUT_PATH = "/opt/gda/input/ingestion.json" + WAREHOUSE = "s3://gda-metadata-warehouse/warehouse" + OBJECT_STORE_ENDPOINT = "http://metadata-object-store:9000" + DATA_PREFIX = WAREHOUSE + "/cultural_heritage/cultural_districts/data/" + + + with open(INPUT_PATH, encoding="utf-8") as handle: + payload = json.load(handle) + + builder = ( + SedonaContext.builder() + .appName("gda-real-feature-iceberg-ingestion") + .master("local[2]") + .config( + "spark.sql.extensions", + "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions", + ) + .config("spark.jars", "/opt/spark/jars-extra/iceberg-aws-bundle-1.6.1.jar") + .config("spark.sql.catalog.rest", "org.apache.iceberg.spark.SparkCatalog") + .config("spark.sql.catalog.rest.type", "rest") + .config( + "spark.sql.catalog.rest.uri", + "http://gravitino-persistence:9001/iceberg", + ) + .config( + "spark.sql.catalog.rest.io-impl", + "org.apache.iceberg.aws.s3.S3FileIO", + ) + .config("spark.sql.catalog.rest.s3.endpoint", OBJECT_STORE_ENDPOINT) + .config("spark.sql.catalog.rest.s3.path-style-access", "true") + .config( + "spark.sql.catalog.rest.s3.access-key-id", + os.environ["AWS_ACCESS_KEY_ID"], + ) + .config( + "spark.sql.catalog.rest.s3.secret-access-key", + os.environ["AWS_SECRET_ACCESS_KEY"], + ) + .config("spark.sql.catalog.rest.client.region", "us-east-1") + .config("spark.sql.catalog.rest.cache-enabled", "false") + .config("spark.sql.shuffle.partitions", "2") + ) + spark = builder.getOrCreate() + sedona = SedonaContext.create(spark) + spark.sparkContext.setLogLevel("WARN") + + input_schema = T.StructType( + [ + T.StructField("BSM", T.StringType(), False), + T.StructField("geometry_wkb_hex", T.StringType(), False), + T.StructField("srid", T.IntegerType(), False), + T.StructField("min_x", T.DoubleType(), False), + T.StructField("min_y", T.DoubleType(), False), + T.StructField("max_x", T.DoubleType(), False), + T.StructField("max_y", T.DoubleType(), False), + T.StructField("row_sha256", T.StringType(), False), + ] + ) + source = ( + spark.createDataFrame(payload["rows"], schema=input_schema) + .withColumn("geometry", F.unhex("geometry_wkb_hex")) + .select( + "BSM", + "geometry", + "srid", + "min_x", + "min_y", + "max_x", + "max_y", + "row_sha256", + ) + ) + source.createOrReplaceTempView("real_feature_source") + + quality = sedona.sql( + """ + WITH spatial AS ( + SELECT *, ST_SetSRID(ST_GeomFromWKB(geometry), srid) AS geom + FROM real_feature_source + ) + SELECT + COUNT(*) AS feature_count, + COUNT(DISTINCT BSM) AS unique_bsm_count, + SUM(CASE WHEN ST_IsValid(geom) THEN 1 ELSE 0 END) AS valid_geometry_count, + SUM(CASE WHEN ST_SRID(geom) = 4490 THEN 1 ELSE 0 END) AS srid_match_count, + SUM(CASE WHEN ST_Area(geom) > 0 THEN 1 ELSE 0 END) AS positive_area_count, + SUM( + CASE WHEN + ABS(ST_XMin(geom) - min_x) <= 1e-12 AND + ABS(ST_YMin(geom) - min_y) <= 1e-12 AND + ABS(ST_XMax(geom) - max_x) <= 1e-12 AND + ABS(ST_YMax(geom) - max_y) <= 1e-12 + THEN 1 ELSE 0 END + ) AS bbox_match_count + FROM spatial + """ + ).collect()[0].asDict() + expected_count = payload["expected_feature_count"] + if any( + quality[key] != expected_count + for key in ( + "feature_count", + "unique_bsm_count", + "valid_geometry_count", + "srid_match_count", + "positive_area_count", + "bbox_match_count", + ) + ): + raise RuntimeError(f"Sedona quality gate failed: {quality}") + + + def table_state(): + rows = spark.sql( + f"SELECT row_sha256 FROM {TABLE} ORDER BY row_sha256" + ).collect() + snapshots = spark.sql( + f"SELECT snapshot_id, parent_id, operation FROM {TABLE}.snapshots " + "ORDER BY committed_at, snapshot_id" + ).collect() + files = spark.sql( + f"SELECT file_path, record_count FROM {TABLE}.files ORDER BY file_path" + ).collect() + return { + "row_sha256": [row["row_sha256"] for row in rows], + "snapshots": [row.asDict() for row in snapshots], + "data_files": [row.asDict() for row in files], + } + + + expected_rows = sorted(payload["expected_row_sha256"]) + + + def apply_once(): + before = table_state() + if not before["row_sha256"] and not before["snapshots"]: + source.coalesce(1).writeTo(TABLE).append() + status = "appended" + mutation_count = 1 + elif before["row_sha256"] == expected_rows and len(before["snapshots"]) == 1: + status = "no_op" + mutation_count = 0 + else: + raise RuntimeError("existing Iceberg table is partial or content-drifted") + spark.catalog.refreshTable(TABLE) + after = table_state() + if ( + after["row_sha256"] != expected_rows + or len(after["snapshots"]) != 1 + or after["snapshots"][0]["operation"] != "append" + or len(after["data_files"]) != 1 + or after["data_files"][0]["record_count"] != expected_count + or not after["data_files"][0]["file_path"].startswith(DATA_PREFIX) + ): + raise RuntimeError("Iceberg content readback does not match the input binding") + return { + "status": status, + "mutation_count": mutation_count, + "row_sha256": after["row_sha256"], + "snapshots": after["snapshots"], + "data_files": after["data_files"], + } + + + first = apply_once() + replay = apply_once() + if first["status"] != "appended" or replay["status"] != "no_op": + raise RuntimeError("real feature ingestion replay boundary failed") + if first != {**replay, "status": "appended", "mutation_count": 1}: + raise RuntimeError("real feature table changed during no-op replay") + + result = { + "schema": "gda.real_feature_ingestion_probe_result.v1", + "plan_sha256": payload["plan_sha256"], + "authorization_sha256": payload["authorization_sha256"], + "source_resource_version_id": payload["source_resource_version_id"], + "source_content_sha256": payload["source_content_sha256"], + "output_resource_version_id": payload["output_resource_version_id"], + "output_content_sha256": payload["output_content_sha256"], + "row_set_sha256": payload["row_set_sha256"], + "spark_version": spark.version, + "sedona_version": "1.9.0", + "iceberg_runtime": "1.6.1", + "table": TABLE, + "quality": quality, + "first_execution": first, + "immediate_replay": replay, + "table_columns": spark.table(TABLE).columns, + "source_payload_recorded": False, + "material_recorded": False, + } + print("GDA_REAL_FEATURE_INGESTION_RESULT=" + json.dumps(result, sort_keys=True)) + spark.stop() +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: real-feature-ingestion-probe + namespace: gda-metadata-spark-object-store +spec: + suspend: true + backoffLimit: 0 + activeDeadlineSeconds: 900 + template: + metadata: + labels: + app.kubernetes.io/name: real-feature-ingestion-probe + app.kubernetes.io/component: real-feature-ingestion + spec: + serviceAccountName: spark-object-store-probe + automountServiceAccountToken: false + nodeSelector: + kubernetes.io/hostname: desktop-worker + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 100 + fsGroup: 100 + seccompProfile: + type: RuntimeDefault + containers: + - name: spark + image: gisdataagent/mmfe-spark-runtime:local + imagePullPolicy: Never + command: ["python", "/opt/gda/probe.py"] + env: + - name: JAVA_HOME + value: /usr/lib/jvm/java-17-openjdk-arm64 + - name: HOME + value: /tmp/spark-home + - name: SPARK_LOCAL_DIRS + value: /tmp/spark-local + - name: PYSPARK_PYTHON + value: python + - name: PYSPARK_SUBMIT_ARGS + value: >- + --jars /opt/spark/jars-extra/iceberg-aws-bundle-1.6.1.jar + --driver-class-path /opt/spark/jars-extra/iceberg-aws-bundle-1.6.1.jar + pyspark-shell + - name: AWS_REGION + value: us-east-1 + - name: AWS_DEFAULT_REGION + value: us-east-1 + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: metadata-object-store-runtime + key: access-key-id + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: metadata-object-store-runtime + key: secret-access-key + resources: + requests: + cpu: "1" + memory: 2Gi + limits: + cpu: "4" + memory: 5Gi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: probe + mountPath: /opt/gda + readOnly: true + - name: input + mountPath: /opt/gda/input + readOnly: true + - name: tmp + mountPath: /tmp + volumes: + - name: probe + configMap: + name: real-feature-ingestion-probe + - name: input + configMap: + name: real-feature-ingestion-input + - name: tmp + emptyDir: {} diff --git a/scripts/metadata-fabric-real-feature-ingestion.sh b/scripts/metadata-fabric-real-feature-ingestion.sh new file mode 100755 index 00000000..c453045a --- /dev/null +++ b/scripts/metadata-fabric-real-feature-ingestion.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +COMMON_GIT_DIR="$(git -C "$ROOT" rev-parse --path-format=absolute --git-common-dir 2>/dev/null || true)" +SHARED_ROOT="" +if [ -n "$COMMON_GIT_DIR" ]; then + SHARED_ROOT="$(cd "$COMMON_GIT_DIR/.." && pwd)" +fi + +if [ -n "${PYTHON:-}" ]; then + : +elif [ -x "$ROOT/.venv/bin/python" ]; then + PYTHON="$ROOT/.venv/bin/python" +elif [ -n "$SHARED_ROOT" ] && [ -x "$SHARED_ROOT/.venv/bin/python" ]; then + PYTHON="$SHARED_ROOT/.venv/bin/python" +else + PYTHON="python" +fi + +cd "$ROOT" +exec "$PYTHON" -m data_agent.metadata_fabric_real_feature_ingestion "$@"