diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 534a4214..74f6bc45 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,7 @@ on: - feat/ar1-metadata-fabric-durable-active-metadata-promotion - feat/ar1-metadata-fabric-object-store-active-metadata-promotion - feat/ar1-metadata-fabric-real-feature-ingestion + - feat/ar1-metadata-fabric-real-feature-ledger-promotion env: PYTHON_VERSION: "3.13" @@ -204,6 +205,9 @@ jobs: - name: Validate metadata fabric real-feature ledger promotion evidence run: python -m data_agent.metadata_fabric_real_feature_ledger_promotion validate + - name: Validate retained real-feature terminal success evidence + run: python -m data_agent.metadata_fabric_retained_real_feature_terminal_success validate + - name: Validate Active Metadata consumer deployment boundary run: python -m data_agent.active_metadata_consumer_deployment validate @@ -248,13 +252,14 @@ jobs: DATABASE_URL: postgresql://postgres:postgres@localhost:5432/gis_agent_test run: python -m pytest data_agent/test_active_metadata_authorization_postgres.py -q - - name: Verify binding reconciliation and real-feature ledger promotion on PostgreSQL + - name: Verify binding reconciliation and real-feature terminal promotion on PostgreSQL env: DATABASE_URL: postgresql://postgres:postgres@localhost:5432/gis_agent_test run: >- python -m pytest data_agent/test_metadata_fabric_active_metadata_binding_reconciliation_postgres.py data_agent/test_metadata_fabric_real_feature_ledger_promotion_postgres.py + data_agent/test_metadata_fabric_retained_real_feature_terminal_success_postgres.py -q - name: Run required platform tests @@ -300,6 +305,7 @@ jobs: data_agent/test_metadata_fabric_object_store_active_metadata_promotion.py \ data_agent/test_metadata_fabric_real_feature_ingestion.py \ data_agent/test_metadata_fabric_real_feature_ledger_promotion.py \ + data_agent/test_metadata_fabric_retained_real_feature_terminal_success.py \ data_agent/test_metadata_fabric_lineage_delivery.py \ data_agent/test_metadata_fabric_provider_identity.py \ data_agent/test_metadata_fabric_gravitino_identity.py \ diff --git a/data_agent/metadata_fabric_retained_real_feature_terminal_success.py b/data_agent/metadata_fabric_retained_real_feature_terminal_success.py new file mode 100644 index 00000000..54d517ec --- /dev/null +++ b/data_agent/metadata_fabric_retained_real_feature_terminal_success.py @@ -0,0 +1,2201 @@ +"""Finalize one retained real-feature staging Run from complete evidence. + +M3-24 keeps the M3-22/M3-23 history immutable. It rebuilds the same bounded +real-feature execution in a retained local staging runtime, persists complete +DolphinScheduler dispatch authorization, requires a fresh material readback, +replaces the executor-created quality candidate with evidence created by the +independent evaluator, and then uses the existing output promoter and database +success finalizer. + +Retained local staging is deliberately not production. Protected workload +identity, production storage, TLS/OIDC, durable catalog operations and tenant +attestation remain separate readiness gates. +""" + +from __future__ import annotations + +import argparse +import hashlib +import io +import json +import secrets +import socket +import subprocess +import threading +import time +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path +from typing import Any, Literal +from urllib.parse import urlsplit +from uuid import UUID, uuid5 + +import pyarrow.parquet as pq +import shapely +from pydantic import BaseModel, ConfigDict, Field, SecretStr, model_validator +from sqlalchemy import create_engine, text + +from . import metadata_fabric_active_metadata_scheduler_delivery as delivery +from . import metadata_fabric_object_store_active_metadata_promotion as m321 +from . import metadata_fabric_real_feature_ingestion as m322 +from . import metadata_fabric_real_feature_ledger_promotion as m323 +from . import metadata_fabric_spark_object_store_interoperability as m310 +from .dolphinscheduler_adapter import ( + DOLPHINSCHEDULER_API_PROFILE, + DOLPHINSCHEDULER_SERVER_VERSION, + DolphinSchedulerAdapter, + DolphinSchedulerClient, + DolphinSchedulerDefinitionBinding, + DolphinSchedulerProfile, + DolphinSchedulerWorkflowSpec, + build_dolphinscheduler_binding_artifact, + compile_dolphinscheduler_workflow, +) +from .platform_authorization import ( + build_approval_artifact, + build_policy_decision_artifact, + validate_run_authorization_evidence, +) +from .platform_contracts import ( + ApprovalRecord, + Artifact, + FrameworkAttemptObservation, + PlatformDefinitionVersion, + PlatformRun, + PolicyDecision, + QualityResult, + ResourceVersion, + RunPolicyReferences, + RunStatus, + RunSuccessEvidence, + SubjectContext, + canonical_json_bytes, + canonical_json_fingerprint, + platform_definition_fingerprint, + quality_result_fingerprint, + run_success_evidence_fingerprint, +) +from .platform_gateway import ( + DefinitionRegistration, + GatewayWriteResult, + PlatformGateway, +) + +CONTRACT_SCHEMA = "gda.retained_real_feature_terminal_success_contract.v1" +RETENTION_SCHEMA = "gda.retained_real_feature_material_observation.v1" +REQUEST_SCHEMA = "gda.retained_real_feature_execution_request.v1" +EVIDENCE_SCHEMA = "gda.retained_real_feature_terminal_success_evidence.v1" +VALIDATION_SCHEMA = "gda.retained_real_feature_terminal_success_validation.v1" +SOURCE_INGESTION_EVIDENCE_SHA256 = m323.SOURCE_EVIDENCE_SHA256 +SOURCE_PROMOTION_EVIDENCE_SHA256 = ( + "f6efea5000791dec1716a8354a8e39a8425b083ca4d409f4bcb61f0e7e03580d" +) +TENANT = m322.TENANT +RUN_ID = m322.RUN_ID +DEFINITION_VERSION_ID = m322.DEFINITION_VERSION_ID +SOURCE_RESOURCE_VERSION_ID = m322.SOURCE_RESOURCE_VERSION_ID +OUTPUT_RESOURCE_VERSION_ID = m322.OUTPUT_RESOURCE_VERSION_ID +RUNNER = m322.WORKLOAD +QUALITY_EVALUATOR = m322.QUALITY_EVALUATOR +POLICY_EVALUATOR = "workload:real-feature-terminal-policy-evaluator" +APPROVER = "human:metadata-platform-owner" +TASK_CODE = 900000000000000024 +CALLBACK_PATH = "/m3-24/execute" +CONTROL_POSTGRES_IMAGE = "postgres:16.10-bookworm" +CONTROL_POSTGRES_IMAGE_ID = ( + "sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74" +) +RETENTION_DAYS = 7 +REPO_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_SOURCE_EVIDENCE_PATH = m322.DEFAULT_EVIDENCE_PATH +DEFAULT_PROMOTION_EVIDENCE_PATH = m323.DEFAULT_EVIDENCE_PATH +DEFAULT_EVIDENCE_PATH = ( + REPO_ROOT + / "docs/evidence/metadata-fabric-retained-real-feature-terminal-success-2026-07-31.json" +) +DEFAULT_WRAPPER_PATH = ( + REPO_ROOT / "scripts/metadata-fabric-retained-real-feature-terminal-success.sh" +) +MIGRATIONS = m323.MIGRATIONS +FALSE_CLAIMS = ( + "source_dataset_committed", + "source_absolute_path_committed", + "source_feature_payload_committed", + "protected_workload_identity_verified", + "durable_catalog_verified", + "production_object_store_verified", + "production_scheduler_verified", + "production_ingestion_verified", + "production_tenant_attestation_verified", + "oidc_verified", + "tls_verified", + "production_ready", +) + + +class RetainedTerminalSuccessError(RuntimeError): + """The retained real-feature terminal-success gate failed closed.""" + + +class _FrozenModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + +class RetainedExecutionRequest(_FrozenModel): + request_schema: Literal[REQUEST_SCHEMA] = Field(default=REQUEST_SCHEMA, alias="schema") + tenant_id: Literal[TENANT] + run_id: UUID + definition_version_id: UUID + source_resource_version_id: UUID + output_resource_version_id: UUID + output_content_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + ingestion_plan_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + retention_id: str = Field(min_length=8, max_length=128, pattern=r"^[a-z0-9-]+$") + request_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + + @model_validator(mode="after") + def _valid_fingerprint(self) -> RetainedExecutionRequest: + if ( + self.run_id != RUN_ID + or self.definition_version_id != DEFINITION_VERSION_ID + or self.source_resource_version_id != SOURCE_RESOURCE_VERSION_ID + or self.output_resource_version_id != OUTPUT_RESOURCE_VERSION_ID + ): + raise ValueError("retained execution request identity does not match") + expected = canonical_json_fingerprint( + self.model_dump(mode="json", by_alias=True, exclude={"request_sha256"}) + ) + if self.request_sha256 != expected: + raise ValueError("retained execution request fingerprint does not match") + return self + + +class RetainedMaterialObservation(_FrozenModel): + observation_schema: Literal[RETENTION_SCHEMA] = Field( + default=RETENTION_SCHEMA, alias="schema" + ) + tenant_id: Literal[TENANT] + run_id: UUID + output_resource_version_id: UUID + output_content_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + storage_uri: str + retention_id: str = Field(min_length=8, max_length=128, pattern=r"^[a-z0-9-]+$") + owner: Literal["team:metadata-platform"] + namespace: str = Field(min_length=1, max_length=253) + namespace_uid: str = Field(min_length=8, max_length=128) + control_database_ref: str = Field(min_length=1, max_length=255) + object_inventory_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + metadata_body_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + row_set_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + snapshot_id: int + feature_count: Literal[20] + data_file_count: Literal[1] + data_size_bytes: int = Field(gt=0) + readable: Literal[True] + source_payload_retained: Literal[False] + materialized_at: datetime + observed_at: datetime + expires_at: datetime + observation_sha256: str = Field(pattern=r"^[0-9a-f]{64}$") + + @model_validator(mode="after") + def _consistent_retention(self) -> RetainedMaterialObservation: + if ( + self.run_id != RUN_ID + or self.output_resource_version_id != OUTPUT_RESOURCE_VERSION_ID + ): + raise ValueError("retained material observation identity does not match") + parts = urlsplit(self.storage_uri) + if parts.scheme != "s3" or not parts.netloc or parts.query or parts.fragment: + raise ValueError("retained material must use a stable S3 URI") + if not self.materialized_at < self.observed_at < self.expires_at: + raise ValueError("retained material timestamps are not ordered") + expected = canonical_json_fingerprint( + self.model_dump(mode="json", by_alias=True, exclude={"observation_sha256"}) + ) + if self.observation_sha256 != expected: + raise ValueError("retained material observation fingerprint does not match") + return self + + +@dataclass(frozen=True) +class TerminalDefinitionBundle: + registration: DefinitionRegistration + definition: PlatformDefinitionVersion + workflow: DolphinSchedulerWorkflowSpec + + +@dataclass(frozen=True) +class TerminalAuthorizationBundle: + source_resource: Any + source_version: ResourceVersion + definition_registration: DefinitionRegistration + output_resource: Any + execution_plan: Artifact + policy_decision: Artifact + approval: Artifact + run: PlatformRun + + +def _run_command(args: list[str], *, timeout: float = 180) -> str: + completed = subprocess.run( + args, + check=False, + capture_output=True, + text=True, + timeout=timeout, + ) + if completed.returncode != 0: + raise RetainedTerminalSuccessError( + f"local runtime command failed: {Path(args[0]).name}" + ) + return completed.stdout.strip() + + +def _free_loopback_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as candidate: + candidate.bind(("127.0.0.1", 0)) + return int(candidate.getsockname()[1]) + + +class RetainedControlPostgres: + """Own one labeled local PostgreSQL container and its retained volume.""" + + def __init__( + self, + retention_id: str, + *, + expires_at: datetime, + password: SecretStr, + ) -> None: + suffix = retention_id.removeprefix("m3-24-")[:24] + self.retention_id = retention_id + self.expires_at = expires_at + self.password = password + self.container_name = f"gda-m3-24-control-{suffix}" + self.volume_name = f"gda-m3-24-control-{suffix}" + self.host_port = _free_loopback_port() + self.database_url = ( + "postgresql://postgres:" + f"{password.get_secret_value()}@127.0.0.1:{self.host_port}/postgres" + ) + self.created = False + + @property + def database_ref(self) -> str: + return f"docker:{self.container_name}/postgres" + + def start(self) -> dict[str, Any]: + image_id = _run_command( + ["docker", "image", "inspect", CONTROL_POSTGRES_IMAGE, "--format", "{{.Id}}"] + ) + if image_id != CONTROL_POSTGRES_IMAGE_ID: + raise RetainedTerminalSuccessError( + "retained control PostgreSQL image identity drifted" + ) + expiry = self.expires_at.isoformat().replace("+00:00", "Z") + labels = [ + "--label", + f"gda.retention-id={self.retention_id}", + "--label", + "gda.owner=team:metadata-platform", + "--label", + f"gda.expires-at={expiry}", + ] + _run_command( + [ + "docker", + "volume", + "create", + *labels, + self.volume_name, + ] + ) + try: + _run_command( + [ + "docker", + "run", + "--detach", + "--name", + self.container_name, + *labels, + "--publish", + f"127.0.0.1:{self.host_port}:5432", + "--mount", + f"source={self.volume_name},target=/var/lib/postgresql/data", + "--env", + f"POSTGRES_PASSWORD={self.password.get_secret_value()}", + CONTROL_POSTGRES_IMAGE, + ] + ) + self.created = True + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + engine = create_engine(self.database_url) + try: + with engine.connect() as connection: + connection.execute(text("SELECT 1")).scalar_one() + break + except Exception: + time.sleep(1) + finally: + engine.dispose() + else: + raise RetainedTerminalSuccessError( + "retained control PostgreSQL did not become ready" + ) + return self.observe() + except BaseException: + self.cleanup() + raise + + def observe(self) -> dict[str, Any]: + state = json.loads( + _run_command( + [ + "docker", + "container", + "inspect", + self.container_name, + "--format", + "{{json .State}}", + ] + ) + ) + volume = json.loads( + _run_command( + [ + "docker", + "volume", + "inspect", + self.volume_name, + "--format", + "{{json .}}", + ] + ) + ) + labels = volume.get("Labels") if isinstance(volume, dict) else None + return { + "database_ref": self.database_ref, + "container_name": self.container_name, + "volume_name": self.volume_name, + "host_port": self.host_port, + "container_running": state.get("Running") is True, + "container_status": state.get("Status"), + "volume_retained": isinstance(volume, dict), + "retention_id": (labels or {}).get("gda.retention-id"), + "owner": (labels or {}).get("gda.owner"), + "expires_at": (labels or {}).get("gda.expires-at"), + "credential_recorded": False, + } + + def cleanup(self) -> None: + if self.created: + subprocess.run( + ["docker", "rm", "--force", self.container_name], + check=False, + capture_output=True, + text=True, + timeout=120, + ) + self.created = False + subprocess.run( + ["docker", "volume", "rm", self.volume_name], + check=False, + capture_output=True, + text=True, + timeout=120, + ) + + +def _authorization_fingerprint(bundle: TerminalAuthorizationBundle) -> str: + return canonical_json_fingerprint( + { + "run": bundle.run.model_dump(mode="json"), + "execution_plan": bundle.execution_plan.model_dump(mode="json"), + "policy_decision": bundle.policy_decision.model_dump(mode="json"), + "approval": bundle.approval.model_dump(mode="json"), + } + ) + + +class RetainedRealFeatureExecutor: + """Perform the one provider mutation invoked by DolphinScheduler.""" + + def __init__( + self, + request: RetainedExecutionRequest, + profile: m322.RealFeatureIngestionProfile, + plan: m322.RealFeatureIngestionPlan, + source: Mapping[str, Any], + runtime: m310.IsolatedSparkObjectStoreRuntime, + rehearsal: m321.ObjectStoreProjectionRehearsal, + *, + endpoint_url: str, + object_store_user: SecretStr, + object_store_material: SecretStr, + authorization_sha256: str, + ) -> None: + self.request = request + self.profile = profile + self.plan = plan + self.source = source + self.runtime = runtime + self.rehearsal = rehearsal + self.endpoint_url = endpoint_url + self.object_store_user = object_store_user + self.object_store_material = object_store_material + self.authorization_sha256 = authorization_sha256 + self.request_count = 0 + self.table_create: dict[str, Any] | None = None + self.spark: dict[str, Any] | None = None + self.store: dict[str, Any] | None = None + self.output_contracts: dict[str, Any] | None = None + self.materialized_at: datetime | None = None + self.source_input_removed = False + self.error_type: str | None = None + self.error_stage: str | None = None + self.stage = "waiting_for_request" + + def execute(self, payload: dict[str, Any]) -> dict[str, Any]: + self.request_count += 1 + try: + observed = RetainedExecutionRequest.model_validate(payload) + if observed != self.request or self.request_count != 1: + raise RetainedTerminalSuccessError( + "retained executor accepts exactly one compiled request" + ) + self.stage = "creating_target_table" + self.table_create = m322.create_target_table( + self.rehearsal, self.profile, self.plan + ) + input_payload = { + **_mapping(self.source.get("payload")), + "plan_sha256": self.plan.ingestion_plan_sha256, + "authorization_sha256": self.authorization_sha256, + } + try: + self.stage = "running_spark_ingestion" + self.spark = m322._run_spark_ingestion( + self.runtime, input_payload=input_payload + ) + self.stage = "observing_object_store" + self.store = m322.observe_ingested_table( + self.runtime, + self.profile, + endpoint_url=self.endpoint_url, + object_store_user=self.object_store_user, + object_store_material=self.object_store_material, + ) + finally: + self.runtime.kubectl.run( + [ + "-n", + self.runtime.profile.cluster.rehearsal_namespace, + "delete", + "configmap", + "real-feature-ingestion-input", + "--ignore-not-found=true", + "--wait=true", + ], + label="retained real feature source input cleanup", + ) + self.source_input_removed = ( + self.runtime.kubectl.get_json( + [ + "-n", + self.runtime.profile.cluster.rehearsal_namespace, + "get", + "configmap", + "real-feature-ingestion-input", + ], + allow_not_found=True, + label="retained source input absence verification", + ) + is None + ) + assert self.spark is not None and self.store is not None + self.stage = "validating_provider_readback" + errors = m322._spark_errors( + self.spark, + self.plan, + {"projection": _mapping(self.source.get("projection"))}, + expected_authorization_sha256=self.authorization_sha256, + ) + errors.extend(m322._object_store_errors(self.store, self.spark, self.profile)) + if errors or not self.source_input_removed: + raise RetainedTerminalSuccessError( + "scheduler-triggered retained ingestion readback failed" + ) + self.materialized_at = datetime.now(UTC) + self.stage = "building_output_contracts" + self.output_contracts = m322.build_output_contracts( + self.plan, + self.spark, + self.store, + created_at=self.materialized_at, + ) + self.stage = "completed" + return { + "schema": "gda.retained_real_feature_execution_response.v1", + "status": "materialized_and_replayed", + "request_sha256": self.request.request_sha256, + "output_content_sha256": self.plan.output_content_sha256, + "source_payload_retained": False, + } + except Exception as exc: + self.error_type = type(exc).__name__ + self.error_stage = self.stage + raise + + +class RetainedExecutionServer: + def __init__(self) -> None: + self.executor: RetainedRealFeatureExecutor | None = None + self.started = False + self.cleanup_verified = False + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 + if self.path != CALLBACK_PATH: + self.send_error(404) + return + try: + length = int(self.headers.get("Content-Length", "0")) + if length <= 0 or length > 32768 or owner.executor is None: + raise ValueError("retained execution request is unavailable") + payload = json.loads(self.rfile.read(length)) + if not isinstance(payload, dict): + raise ValueError("retained execution request must be an object") + response = owner.executor.execute(payload) + except Exception: + self.send_error(500) + return + body = json.dumps( + response, + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, _format: str, *args: object) -> None: + return + + self._server = HTTPServer(("0.0.0.0", 0), Handler) + self._thread = threading.Thread( + target=self._server.serve_forever, + name="gda-m3-24-retained-real-feature-executor", + daemon=True, + ) + + @property + def callback_url(self) -> str: + return f"http://host.docker.internal:{self._server.server_port}{CALLBACK_PATH}" + + def start(self) -> None: + if self.executor is None: + raise RetainedTerminalSuccessError( + "retained execution server requires an executor" + ) + self._thread.start() + self.started = True + + def stop(self) -> bool: + if self.started: + self._server.shutdown() + self._thread.join(timeout=30) + self._server.server_close() + self.cleanup_verified = not self._thread.is_alive() + return self.cleanup_verified + + +def _mapping(value: Any) -> dict[str, Any]: + return dict(value) if isinstance(value, Mapping) else {} + + +def _load_json_object(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise RetainedTerminalSuccessError(f"{path.name} must contain an object") + return value + + +def _file_record(path: Path) -> dict[str, str | None]: + relative = path.resolve().relative_to(REPO_ROOT).as_posix() + return { + "path": relative, + "sha256": ( + hashlib.sha256(path.read_bytes()).hexdigest() if path.is_file() else None + ), + } + + +def _mark_namespace_retained( + runtime: m310.IsolatedSparkObjectStoreRuntime, + *, + retention_id: str, + expires_at: datetime, +) -> dict[str, Any]: + namespace = runtime.profile.cluster.rehearsal_namespace + expiry = expires_at.isoformat().replace("+00:00", "Z") + runtime.kubectl.run( + [ + "label", + "namespace", + namespace, + f"gda.gisdataagent.io/retention-id={retention_id}", + "gda.gisdataagent.io/owner=metadata-platform", + "--overwrite", + ], + label="retained namespace ownership labels", + ) + runtime.kubectl.run( + [ + "annotate", + "namespace", + namespace, + f"gda.gisdataagent.io/expires-at={expiry}", + "gda.gisdataagent.io/cleanup-command=" + "metadata-fabric-retained-real-feature-terminal-success cleanup", + "--overwrite", + ], + label="retained namespace lifecycle annotations", + ) + observed = runtime.kubectl.get_json( + ["get", "namespace", namespace], label="retained namespace readback" + ) + assert observed is not None + metadata = _mapping(observed.get("metadata")) + labels = _mapping(metadata.get("labels")) + annotations = _mapping(metadata.get("annotations")) + result = { + "name": metadata.get("name"), + "uid": metadata.get("uid"), + "retention_id": labels.get("gda.gisdataagent.io/retention-id"), + "owner": labels.get("gda.gisdataagent.io/owner"), + "expires_at": annotations.get("gda.gisdataagent.io/expires-at"), + "cleanup_command_recorded": bool( + annotations.get("gda.gisdataagent.io/cleanup-command") + ), + } + if result != { + "name": namespace, + "uid": metadata.get("uid"), + "retention_id": retention_id, + "owner": "metadata-platform", + "expires_at": expiry, + "cleanup_command_recorded": True, + }: + raise RetainedTerminalSuccessError( + "retained namespace lifecycle readback drifted" + ) + return result + + +def independently_evaluate_retained_parquet( + runtime: m310.IsolatedSparkObjectStoreRuntime, + profile: m322.RealFeatureIngestionProfile, + plan: m322.RealFeatureIngestionPlan, + source: Mapping[str, Any], + store: Mapping[str, Any], + *, + endpoint_url: str, + object_store_user: SecretStr, + object_store_material: SecretStr, +) -> dict[str, Any]: + data_keys = list(store.get("data_keys") or []) + if len(data_keys) != 1: + raise RetainedTerminalSuccessError( + "independent evaluator requires exactly one Parquet data file" + ) + client = runtime._s3_client( + endpoint_url=endpoint_url, + object_store_user=object_store_user, + object_store_material=object_store_material, + ) + try: + response = client.get_object(Bucket=profile.target.bucket, Key=data_keys[0]) + body = response["Body"].read() + finally: + client.close() + table = pq.read_table(io.BytesIO(body)) + expected_columns = list(m322.SPARK_COLUMNS) + if table.column_names != expected_columns or table.num_rows != plan.expected_feature_count: + raise RetainedTerminalSuccessError( + "independent Parquet schema or feature count drifted" + ) + columns = table.to_pydict() + row_hashes: list[str] = [] + valid_count = 0 + non_empty_count = 0 + positive_area_count = 0 + bbox_match_count = 0 + geometry_z_count = 0 + for index in range(table.num_rows): + geometry_bytes = bytes(columns["geometry"][index]) + geometry = shapely.from_wkb(geometry_bytes) + valid_count += int(bool(shapely.is_valid(geometry))) + non_empty_count += int(not bool(shapely.is_empty(geometry))) + positive_area_count += int(float(shapely.area(geometry)) > 0) + geometry_z_count += int(bool(shapely.has_z(geometry))) + min_x, min_y, max_x, max_y = shapely.bounds(geometry) + expected_bounds = ( + float(columns["min_x"][index]), + float(columns["min_y"][index]), + float(columns["max_x"][index]), + float(columns["max_y"][index]), + ) + bbox_match_count += int( + all( + abs(float(actual) - expected) <= 1e-12 + for actual, expected in zip( + (min_x, min_y, max_x, max_y), expected_bounds, strict=True + ) + ) + ) + stable = { + "BSM": str(columns["BSM"][index]), + "geometry_wkb_hex": geometry_bytes.hex(), + "srid": int(columns["srid"][index]), + "min_x": expected_bounds[0], + "min_y": expected_bounds[1], + "max_x": expected_bounds[2], + "max_y": expected_bounds[3], + } + row_hash = canonical_json_fingerprint(stable) + if row_hash != str(columns["row_sha256"][index]): + raise RetainedTerminalSuccessError( + "independent Parquet row fingerprint drifted" + ) + row_hashes.append(row_hash) + projection = _mapping(source.get("projection")) + expected_hashes = sorted(str(value) for value in projection.get("row_sha256") or []) + metrics = { + "feature_count": table.num_rows, + "unique_bsm_count": len(set(str(value) for value in columns["BSM"])), + "valid_geometry_count": valid_count, + "non_empty_geometry_count": non_empty_count, + "geometry_z_count": geometry_z_count, + "srid_match_count": sum(int(value) == 4490 for value in columns["srid"]), + "positive_area_count": positive_area_count, + "bbox_match_count": bbox_match_count, + "row_fingerprint_match_count": sum( + left == right + for left, right in zip(sorted(row_hashes), expected_hashes, strict=True) + ), + } + if any(value != plan.expected_feature_count for value in metrics.values()): + raise RetainedTerminalSuccessError( + "independent retained Parquet quality gate failed" + ) + return { + "metrics": metrics, + "data_key_sha256": hashlib.sha256(data_keys[0].encode()).hexdigest(), + "data_body_sha256": hashlib.sha256(body).hexdigest(), + "data_size_bytes": len(body), + "row_set_sha256": plan.row_set_sha256, + "feature_payload_recorded": False, + "identifier_values_recorded": False, + "geometry_values_recorded": False, + } + + +def build_execution_request( + plan: m322.RealFeatureIngestionPlan, *, retention_id: str +) -> RetainedExecutionRequest: + values = { + "tenant_id": TENANT, + "run_id": RUN_ID, + "definition_version_id": DEFINITION_VERSION_ID, + "source_resource_version_id": SOURCE_RESOURCE_VERSION_ID, + "output_resource_version_id": OUTPUT_RESOURCE_VERSION_ID, + "output_content_sha256": plan.output_content_sha256, + "ingestion_plan_sha256": plan.ingestion_plan_sha256, + "retention_id": retention_id, + } + stable = { + "schema": REQUEST_SCHEMA, + **{ + key: str(value) if isinstance(value, UUID) else value + for key, value in values.items() + }, + } + return RetainedExecutionRequest( + **values, + request_sha256=canonical_json_fingerprint(stable), + ) + + +def _workflow_document( + callback_url: str, request: RetainedExecutionRequest +) -> dict[str, Any]: + request_json = json.dumps( + request.model_dump(mode="json", by_alias=True), + ensure_ascii=True, + sort_keys=True, + separators=(",", ":"), + ) + quoted_request = request_json.replace("'", "'\"'\"'") + quoted_url = callback_url.replace("'", "'\"'\"'") + raw_script = ( + "curl --fail --silent --show-error --max-time 1200 " + "--request POST --header 'Content-Type: application/json' " + f"--data-binary '{quoted_request}' '{quoted_url}'" + ) + task = { + "code": TASK_CODE, + "name": "ingest_retained_real_feature_slice", + "version": 1, + "description": "Ingest one authorized real feature slice into retained staging", + "delayTime": 0, + "taskType": "SHELL", + "taskParams": { + "localParams": [], + "rawScript": raw_script, + "resourceList": [], + "dependence": {}, + "conditionResult": {"successNode": [], "failedNode": []}, + "waitStartTimeout": {}, + }, + "flag": "YES", + "taskPriority": "MEDIUM", + "workerGroup": "default", + "environmentCode": -1, + "failRetryTimes": 0, + "failRetryInterval": 1, + "timeoutFlag": "OPEN", + "timeoutNotifyStrategy": "WARN", + "timeout": 1260, + } + return { + "dolphinscheduler": { + "name": "gda_retained_real_feature_terminal_success_v1", + "description": "Authorized retained real-feature staging ingestion", + "task_definitions": [task], + "task_relations": [ + { + "name": "", + "preTaskCode": 0, + "preTaskVersion": 0, + "postTaskCode": TASK_CODE, + "postTaskVersion": 1, + "conditionType": "NONE", + "conditionParams": {}, + } + ], + "locations": [{"taskCode": TASK_CODE, "x": 160, "y": 100}], + "global_params": [], + "timeout_seconds": 1320, + "execution_type": "PARALLEL", + } + } + + +def build_terminal_definition( + callback_url: str, + request: RetainedExecutionRequest, + *, + created_at: datetime, +) -> TerminalDefinitionBundle: + definition_urn = f"gda://{TENANT}/definition/real-feature-ingestion" + document = _workflow_document(callback_url, request) + input_contract = { + "source_resource_version_id": str(SOURCE_RESOURCE_VERSION_ID), + "semantic_type": "gis.cultural_districts", + "execution_request_sha256": request.request_sha256, + } + output_contract = { + "output_resource_version_id": str(OUTPUT_RESOURCE_VERSION_ID), + "retained_staging_material": True, + "independent_quality_evidence": True, + "source_to_output_lineage": True, + "platform_run_terminal_success": True, + } + definition_sha = platform_definition_fingerprint( + orchestration_class="dataops", + capability_id=m322.ACTION, + portability_class="provider_native", + definition_document=document, + input_contract=input_contract, + output_contract=output_contract, + ) + resource = m323.Resource( + tenant_id=TENANT, + resource_urn=definition_urn, + resource_kind="definition", + authority_system="gda", + authority_locator="definition/real-feature-ingestion", + owner_ref="team:metadata-platform", + ) + version = ResourceVersion( + tenant_id=TENANT, + resource_urn=definition_urn, + resource_version_id=DEFINITION_VERSION_ID, + version_key="dolphinscheduler-3.4.2-retained-real-feature-v1", + content_sha256=definition_sha, + authority_version_ref={ + "api_profile": DOLPHINSCHEDULER_API_PROFILE, + "server_version": DOLPHINSCHEDULER_SERVER_VERSION, + "execution_request_sha256": request.request_sha256, + }, + created_by="workload:metadata-definition-registrar", + created_at=created_at, + ) + definition = PlatformDefinitionVersion( + tenant_id=TENANT, + definition_urn=definition_urn, + definition_version_id=DEFINITION_VERSION_ID, + orchestration_class="dataops", + capability_id=m322.ACTION, + portability_class="provider_native", + definition_document=document, + input_contract=input_contract, + output_contract=output_contract, + definition_sha256=definition_sha, + ) + registration = DefinitionRegistration( + resource=resource, + resource_version=version, + definition=definition, + ) + return TerminalDefinitionBundle( + registration=registration, + definition=definition, + workflow=compile_dolphinscheduler_workflow(definition), + ) + + +def build_terminal_authorization( + source: Mapping[str, Any], + definition_bundle: TerminalDefinitionBundle, + binding: DolphinSchedulerDefinitionBinding, + *, + authorized_at: datetime, +) -> TerminalAuthorizationBundle: + base_promotion = m323.build_promotion(source) + prerequisites = m323.build_prerequisites(source, base_promotion) + if ( + binding.definition_version_id != DEFINITION_VERSION_ID + or binding.compiled_sha256 != definition_bundle.workflow.compiled_sha256 + ): + raise RetainedTerminalSuccessError( + "DolphinScheduler binding does not match the terminal definition" + ) + execution_plan = build_dolphinscheduler_binding_artifact( + binding, + created_by=RUNNER, + created_at=authorized_at - timedelta(seconds=3), + ) + subject = SubjectContext( + tenant_id=TENANT, + subject_id=RUNNER.removeprefix("workload:"), + subject_type="workload", + roles=("spatial_ingestion_executor",), + purpose="ingest and terminalize one retained real feature staging slice", + ) + decision = PolicyDecision( + tenant_id=TENANT, + run_id=RUN_ID, + subject_context=subject, + action="dolphinscheduler.dispatch", + definition_version_id=DEFINITION_VERSION_ID, + resource_version_ids=(DEFINITION_VERSION_ID, SOURCE_RESOURCE_VERSION_ID), + execution_plan_artifact_id=execution_plan.artifact_id, + effect="allow", + policy_version_ref="policy://gda/metadata-fabric/retained-real-feature/v1", + evaluator_subject=POLICY_EVALUATOR, + requires_approval=True, + decided_at=authorized_at - timedelta(seconds=3), + expires_at=authorized_at + timedelta(days=30), + ) + policy = build_policy_decision_artifact(decision) + approval = build_approval_artifact( + ApprovalRecord( + tenant_id=TENANT, + run_id=RUN_ID, + definition_version_id=DEFINITION_VERSION_ID, + policy_decision_artifact_id=policy.artifact_id, + policy_decision_sha256=policy.content_sha256, + verdict="approved", + approver_subject=APPROVER, + reason="Approve one bounded retained local staging ingestion.", + decided_at=authorized_at - timedelta(seconds=2), + expires_at=authorized_at + timedelta(days=7), + ) + ) + run = PlatformRun( + tenant_id=TENANT, + run_id=RUN_ID, + definition_version_id=DEFINITION_VERSION_ID, + orchestration_class="dataops", + subject_context=subject, + input_bindings=( + { + "binding_name": "source_dataset", + "resource_version_id": SOURCE_RESOURCE_VERSION_ID, + "semantic_type": "gis.cultural_districts", + }, + ), + idempotency_key=( + "retained-real-feature-ingestion:" + f"{base_promotion.output_resource_version.content_sha256}" + ), + policy_refs=RunPolicyReferences( + policy_decision_artifact_id=policy.artifact_id, + approval_artifact_id=approval.artifact_id, + ), + config_fingerprint=definition_bundle.definition.definition_sha256, + submitted_at=authorized_at - timedelta(seconds=1), + ) + validate_run_authorization_evidence( + run, + policy, + approval, + execution_plan, + at=authorized_at, + expected_action="dolphinscheduler.dispatch", + ) + return TerminalAuthorizationBundle( + source_resource=prerequisites.source_resource, + source_version=prerequisites.source_version, + definition_registration=definition_bundle.registration, + output_resource=prerequisites.output_resource, + execution_plan=execution_plan, + policy_decision=policy, + approval=approval, + run=run, + ) + + +def build_retained_material_observation(**values: Any) -> RetainedMaterialObservation: + stable = { + "schema": RETENTION_SCHEMA, + **{ + key: value.isoformat().replace("+00:00", "Z") + if isinstance(value, datetime) + else str(value) + if isinstance(value, UUID) + else value + for key, value in values.items() + }, + } + return RetainedMaterialObservation( + **values, + observation_sha256=canonical_json_fingerprint(stable), + ) + + +def build_terminal_promotion( + source: Mapping[str, Any], + retention: RetainedMaterialObservation, +) -> m323.RunOutputLedgerPromotion: + base = m323.build_promotion(source) + if ( + retention.output_content_sha256 + != base.output_resource_version.content_sha256 + or retention.storage_uri != base.output_artifact.storage_uri + or retention.row_set_sha256 != base.output_artifact.manifest.get("row_set_sha256") + or retention.feature_count != base.output_artifact.manifest.get("feature_count") + ): + raise RetainedTerminalSuccessError( + "retained material does not bind the checked real-feature output" + ) + output_version = base.output_resource_version.model_copy( + update={ + "authority_version_ref": { + **base.output_resource_version.authority_version_ref, + "snapshot_id": retention.snapshot_id, + "retention_id": retention.retention_id, + "namespace_uid": retention.namespace_uid, + "object_inventory_sha256": retention.object_inventory_sha256, + "retention_expires_at": retention.expires_at.isoformat(), + }, + "created_at": retention.materialized_at, + } + ) + output_artifact = base.output_artifact.model_copy( + update={ + "manifest": { + **base.output_artifact.manifest, + "snapshot_id": retention.snapshot_id, + "data_file_count": retention.data_file_count, + "retention_id": retention.retention_id, + "retention_observation_sha256": retention.observation_sha256, + "retention_expires_at": retention.expires_at.isoformat(), + }, + "size_bytes": retention.data_size_bytes, + "created_at": retention.materialized_at, + } + ) + metrics = { + **base.quality_result.metrics, + "independent_material_readback": True, + "retained_feature_count": retention.feature_count, + "retained_data_file_count": retention.data_file_count, + "retained_row_set_sha256": retention.row_set_sha256, + } + quality_manifest = { + "schema": RETENTION_SCHEMA, + "rule_version_ref": "quality://gda/spatial/retained-real-feature/v1", + "metrics": metrics, + "retention_observation": retention.model_dump(mode="json", by_alias=True), + } + quality_artifact_id = uuid5( + RUN_ID, f"retained-quality-evidence:{retention.observation_sha256}" + ) + quality_artifact = Artifact( + tenant_id=TENANT, + artifact_id=quality_artifact_id, + artifact_key=f"retained-real-feature-quality:{quality_artifact_id}", + artifact_role="evidence", + storage_uri=( + f"postgresql://gda-control/quality-evidence/{TENANT}/{quality_artifact_id}" + ), + media_type="application/vnd.gda.retained-real-feature-quality+json", + content_sha256=canonical_json_fingerprint(quality_manifest), + size_bytes=len(canonical_json_bytes(quality_manifest)), + run_id=RUN_ID, + resource_version_id=OUTPUT_RESOURCE_VERSION_ID, + manifest=quality_manifest, + created_by=QUALITY_EVALUATOR, + created_at=retention.observed_at, + ) + quality_sha = quality_result_fingerprint( + tenant_id=TENANT, + run_id=RUN_ID, + resource_version_id=OUTPUT_RESOURCE_VERSION_ID, + rule_version_ref="quality://gda/spatial/retained-real-feature/v1", + verdict="passed", + metrics=metrics, + evidence_artifact_id=quality_artifact_id, + evaluated_by=QUALITY_EVALUATOR, + evaluated_at=retention.observed_at, + ) + quality = QualityResult( + tenant_id=TENANT, + quality_result_id=uuid5(RUN_ID, f"retained-quality:{quality_sha}"), + run_id=RUN_ID, + resource_version_id=OUTPUT_RESOURCE_VERSION_ID, + rule_version_ref="quality://gda/spatial/retained-real-feature/v1", + verdict="passed", + metrics=metrics, + evidence_artifact_id=quality_artifact_id, + result_sha256=quality_sha, + evaluated_by=QUALITY_EVALUATOR, + evaluated_at=retention.observed_at, + ) + lineage_facets = { + **base.lineage_event.facets, + "retention_id": retention.retention_id, + "retention_observation_sha256": retention.observation_sha256, + } + output_artifact = output_artifact.model_copy( + update={"manifest": {**output_artifact.manifest, **lineage_facets}} + ) + lineage_values = { + "event_type": base.lineage_event.event_type.value, + "source_resource_version_id": str(SOURCE_RESOURCE_VERSION_ID), + "target_resource_version_id": str(OUTPUT_RESOURCE_VERSION_ID), + "run_id": str(RUN_ID), + "definition_version_id": str(DEFINITION_VERSION_ID), + "artifact_id": str(output_artifact.artifact_id), + "producer": RUNNER, + "facets": lineage_facets, + "occurred_at": retention.materialized_at.isoformat().replace("+00:00", "Z"), + } + lineage_sha = canonical_json_fingerprint(lineage_values) + lineage = base.lineage_event.model_copy( + update={ + "lineage_event_id": uuid5(RUN_ID, f"retained-lineage:{lineage_sha}"), + "event_sha256": lineage_sha, + "facets": lineage_facets, + "occurred_at": retention.materialized_at, + } + ) + return m323.RunOutputLedgerPromotion( + authority_resource=base.authority_resource, + output_resource_version=output_version, + output_artifact=output_artifact, + quality_evidence_artifact=quality_artifact, + quality_result=quality, + lineage_event=lineage, + ) + + +def register_terminal_authorization( + gateway: PlatformGateway, bundle: TerminalAuthorizationBundle +) -> None: + gateway.register_resource(bundle.source_resource) + gateway.register_resource_version(bundle.source_version) + gateway.register_definition(bundle.definition_registration) + gateway.register_resource(bundle.output_resource) + for artifact in ( + bundle.execution_plan, + bundle.policy_decision, + bundle.approval, + ): + gateway.record_artifact(artifact) + gateway.submit_run(bundle.run) + + +def build_success_evidence( + promotion: m323.RunOutputLedgerPromotion, + observation: FrameworkAttemptObservation, +) -> RunSuccessEvidence: + values = { + "tenant_id": TENANT, + "run_id": RUN_ID, + "attempt_observation_id": observation.observation_id, + "output_artifact_id": promotion.output_artifact.artifact_id, + "quality_result_id": promotion.quality_result.quality_result_id, + "lineage_event_id": promotion.lineage_event.lineage_event_id, + } + return RunSuccessEvidence( + **values, + evidence_sha256=run_success_evidence_fingerprint(**values), + ) + + +class RetainedTerminalSuccessCoordinator: + """Gate promotion/finalization on a fresh, independently verified readback.""" + + def __init__( + self, + gateway: PlatformGateway, + *, + material_probe: Callable[[RetainedMaterialObservation], bool], + ) -> None: + self.gateway = gateway + self.material_probe = material_probe + self.promoter = m323.RunOutputLedgerPromoter(gateway) + + def _verify_terminal_replay( + self, promotion: m323.RunOutputLedgerPromotion + ) -> GatewayWriteResult: + with self.gateway._transaction(TENANT) as connection: + stored = m323.RunOutputLedgerPromotion( + authority_resource=self.gateway._load_resource( + connection, + TENANT, + promotion.authority_resource.resource_urn, + ), + output_resource_version=self.gateway._load_resource_version( + connection, + TENANT, + promotion.output_resource_version.resource_version_id, + ), + output_artifact=self.gateway._load_artifact( + connection, + TENANT, + promotion.output_artifact.artifact_id, + ), + quality_evidence_artifact=self.gateway._load_artifact( + connection, + TENANT, + promotion.quality_evidence_artifact.artifact_id, + ), + quality_result=self.gateway._load_quality_result( + connection, + TENANT, + promotion.quality_result.quality_result_id, + ), + lineage_event=self.gateway._load_lineage( + connection, + TENANT, + promotion.lineage_event.lineage_event_id, + ), + ) + if stored != promotion: + raise RetainedTerminalSuccessError( + "terminal replay facts differ from the successful verdict" + ) + return GatewayWriteResult(stored, False) + + def finalize( + self, + promotion: m323.RunOutputLedgerPromotion, + retention: RetainedMaterialObservation, + observation: FrameworkAttemptObservation, + *, + reason: str = "retained staging output passed terminal evidence gate", + ) -> tuple[GatewayWriteResult, PlatformRun]: + if ( + observation.tenant_id != TENANT + or observation.run_id != RUN_ID + or observation.framework_kind.value != "dolphinscheduler" + or observation.observed_state.lower() != "success" + or observation.evidence.get("provider_state") != "SUCCESS" + or observation.evidence.get("api_profile") + != DOLPHINSCHEDULER_API_PROFILE + or observation.evidence.get("server_version") + != DOLPHINSCHEDULER_SERVER_VERSION + or observation.observation_sha256 + != canonical_json_fingerprint(observation.evidence) + ): + raise RetainedTerminalSuccessError( + "terminal finalization requires a DolphinScheduler success observation" + ) + if ( + promotion.quality_evidence_artifact.created_by != QUALITY_EVALUATOR + or promotion.quality_result.evaluated_by != QUALITY_EVALUATOR + or promotion.quality_result.evidence_artifact_id + != promotion.quality_evidence_artifact.artifact_id + or promotion.quality_evidence_artifact.manifest.get( + "retention_observation" + ) + != retention.model_dump(mode="json", by_alias=True) + ): + raise RetainedTerminalSuccessError( + "quality evidence was not created by the independent evaluator" + ) + if not self.material_probe(retention): + raise RetainedTerminalSuccessError( + "retained staging material is absent, expired, or unreadable" + ) + run = self.gateway.get_run(TENANT, RUN_ID) + if run.status == RunStatus.SUCCEEDED: + promoted = self._verify_terminal_replay(promotion) + elif run.status in {RunStatus.RUNNING, RunStatus.RECONCILING}: + promoted = self.promoter.promote(promotion) + else: + raise RetainedTerminalSuccessError( + "terminal finalization requires a running or reconciling Run" + ) + succeeded = self.gateway.finalize_run_success( + build_success_evidence(promotion, observation), + expected_state_version=run.state_version, + actor_subject=RUNNER, + reason=reason, + ) + return promoted, succeeded + + +def _live_material_probe( + retention: RetainedMaterialObservation, + *, + runtime: m310.IsolatedSparkObjectStoreRuntime, + profile: m322.RealFeatureIngestionProfile, + control: RetainedControlPostgres, + endpoint_url: str, + object_store_user: SecretStr, + object_store_material: SecretStr, +) -> bool: + if datetime.now(UTC) >= retention.expires_at: + return False + namespace = runtime.kubectl.get_json( + ["get", "namespace", retention.namespace], + allow_not_found=True, + label="terminal retained namespace probe", + ) + if namespace is None: + return False + metadata = _mapping(namespace.get("metadata")) + labels = _mapping(metadata.get("labels")) + annotations = _mapping(metadata.get("annotations")) + if ( + metadata.get("uid") != retention.namespace_uid + or labels.get("gda.gisdataagent.io/retention-id") != retention.retention_id + or labels.get("gda.gisdataagent.io/owner") != "metadata-platform" + or annotations.get("gda.gisdataagent.io/expires-at") + != retention.expires_at.isoformat().replace("+00:00", "Z") + ): + return False + source_input = runtime.kubectl.get_json( + [ + "-n", + retention.namespace, + "get", + "configmap", + "real-feature-ingestion-input", + ], + allow_not_found=True, + label="terminal retained source input probe", + ) + if source_input is not None: + return False + control_state = control.observe() + if ( + control_state.get("database_ref") != retention.control_database_ref + or control_state.get("container_running") is not True + or control_state.get("volume_retained") is not True + or control_state.get("retention_id") != retention.retention_id + ): + return False + store = m322.observe_ingested_table( + runtime, + profile, + endpoint_url=endpoint_url, + object_store_user=object_store_user, + object_store_material=object_store_material, + ) + latest = _mapping(store.get("latest_metadata")) + return ( + store.get("object_inventory_sha256") == retention.object_inventory_sha256 + and latest.get("body_sha256") == retention.metadata_body_sha256 + and latest.get("current_snapshot_id") == retention.snapshot_id + and len(store.get("data_keys") or []) == retention.data_file_count + ) + + +def _ledger_counts(engine: Any) -> dict[str, int]: + with engine.connect() as connection: + row = connection.execute( + text( + """ + SELECT + (SELECT count(*) FROM gda_control.artifact + WHERE tenant_id = :tenant_id) AS artifacts, + (SELECT count(*) FROM gda_control.artifact + WHERE tenant_id = :tenant_id + AND artifact_role = 'execution_plan') AS execution_plans, + (SELECT count(*) FROM gda_control.artifact + WHERE tenant_id = :tenant_id + AND media_type = 'application/vnd.gda.policy-decision+json') + AS policy_decisions, + (SELECT count(*) FROM gda_control.artifact + WHERE tenant_id = :tenant_id + AND media_type = 'application/vnd.gda.approval+json') AS approvals, + (SELECT count(*) FROM gda_control.artifact + WHERE tenant_id = :tenant_id + AND run_id = :run_id + AND created_by = :quality_evaluator) AS evaluator_evidence, + (SELECT count(*) FROM gda_control.framework_attempt_observation + WHERE tenant_id = :tenant_id AND run_id = :run_id) AS attempts, + (SELECT count(*) FROM gda_control.quality_result + WHERE tenant_id = :tenant_id AND run_id = :run_id) AS quality_results, + (SELECT count(*) FROM gda_control.lineage_event + WHERE tenant_id = :tenant_id AND run_id = :run_id) AS lineage_events, + (SELECT count(*) FROM gda_control.platform_run_event + WHERE tenant_id = :tenant_id AND run_id = :run_id) AS run_events + """ + ), + { + "tenant_id": TENANT, + "run_id": RUN_ID, + "quality_evaluator": QUALITY_EVALUATOR, + }, + ).mappings().one() + return {key: int(value) for key, value in row.items()} + + +def run_live_rehearsal( + *, + profile_path: Path, + shapefile_path: Path, + ogrinfo_path: Path, + proj_data_path: Path | None, + scheduler_admin_password: SecretStr, + scheduler_readiness_timeout_seconds: float = 240, + terminal_timeout_seconds: float = 1500, +) -> dict[str, Any]: + contract = build_contract_report() + if contract.get("status") != "valid": + raise RetainedTerminalSuccessError("M3-24 static contract is invalid") + profile = m322.load_profile(profile_path) + predecessor, runtime_profile = m322._load_dependencies(profile) + source = m322.build_source_input( + profile, + predecessor, + shapefile_path=shapefile_path, + ogrinfo_path=ogrinfo_path, + proj_data_path=proj_data_path, + ) + checked_source = _load_json_object(DEFAULT_SOURCE_EVIDENCE_PATH) + if ( + source.get("inventory") != checked_source.get("dataset_bundle") + or source.get("projection") != checked_source.get("source_projection") + ): + raise RetainedTerminalSuccessError( + "live source does not match the checked M3-22 dataset projection" + ) + + retention_id = f"m3-24-{secrets.token_hex(8)}" + started_at = datetime.now(UTC) + expires_at = started_at + timedelta(days=RETENTION_DAYS) + admin_material = SecretStr(secrets.token_urlsafe(24)) + database_material = SecretStr(secrets.token_urlsafe(24)) + user_material = SecretStr(secrets.token_urlsafe(24)) + object_store_user = SecretStr("gda" + secrets.token_hex(8)) + object_store_material = SecretStr(secrets.token_urlsafe(32)) + control_password = SecretStr(secrets.token_urlsafe(32)) + runtime = m310.IsolatedSparkObjectStoreRuntime(runtime_profile) + control = RetainedControlPostgres( + retention_id, + expires_at=expires_at, + password=control_password, + ) + server = RetainedExecutionServer() + object_forward: Any = None + gravitino_forward: Any = None + rehearsal: m321.ObjectStoreProjectionRehearsal | None = None + engine: Any = None + client: DolphinSchedulerClient | None = None + retained = False + object_forward_stopped = False + gravitino_forward_stopped = False + namespace_retention: dict[str, Any] | None = None + control_state: dict[str, Any] | None = None + scheduler: delivery.EphemeralDolphinScheduler | None = None + try: + initial_runtime = runtime.start( + admin_material=admin_material, + database_material=database_material, + object_store_user=object_store_user, + object_store_material=object_store_material, + ) + namespace_retention = _mark_namespace_retained( + runtime, + retention_id=retention_id, + expires_at=expires_at, + ) + cluster = runtime.kubectl.get_json( + ["get", "namespace", "kube-system"], + label="retained real feature cluster identity", + ) + assert cluster is not None + cluster_uid = str(_mapping(_mapping(cluster).get("metadata")).get("uid")) + runtime_binding = m321._provider_runtime_binding( + initial_runtime, + cluster_uid=cluster_uid, + target=profile.target, + ) + plan = m322.build_ingestion_plan( + profile, predecessor, source, runtime_binding + ) + request = build_execution_request(plan, retention_id=retention_id) + definition_bundle = build_terminal_definition( + server.callback_url, + request, + created_at=datetime.now(UTC), + ) + + object_forward = m321.provider_metrics._PortForward( + kubectl="kubectl", + context=runtime_profile.cluster.context, + namespace=runtime_profile.cluster.rehearsal_namespace, + service=runtime_profile.runtime.object_store_service, + target_port=runtime_profile.runtime.object_store_service_port, + ) + object_forward.start() + endpoint_url = f"http://127.0.0.1:{object_forward.local_port}" + object_store_prepared = runtime.prepare_object_store( + endpoint_url=endpoint_url, + object_store_user=object_store_user, + object_store_material=object_store_material, + ) + gravitino_forward = m321.provider_metrics._PortForward( + kubectl="kubectl", + context=runtime_profile.cluster.context, + namespace=runtime_profile.cluster.rehearsal_namespace, + service=runtime_profile.runtime.service, + target_port=runtime_profile.runtime.gravitino_service_port, + ) + gravitino_forward.start() + rehearsal = m321.ObjectStoreProjectionRehearsal( + base_url=f"http://127.0.0.1:{gravitino_forward.local_port}/api", + admin_name=profile.identity.service_admin, + admin_material=admin_material, + ) + bootstrap = rehearsal.bootstrap( + profile, + database_material=database_material, + user_material=user_material, + object_store_user=object_store_user, + object_store_material=object_store_material, + ) + + control_state = control.start() + engine = create_engine(control.database_url) + _apply_migrations(engine) + gateway = PlatformGateway(engine) + scheduler = delivery.EphemeralDolphinScheduler( + scheduler_admin_password, + readiness_timeout=scheduler_readiness_timeout_seconds, + ) + with scheduler: + project_code, access_token = scheduler.provision_project() + scheduler_profile = DolphinSchedulerProfile( + base_url=scheduler.base_url, + access_token=access_token, + project_code=project_code, + workload_subject=RUNNER, + policy_evaluator_subject=POLICY_EVALUATOR, + tenant_code="default", + worker_group="default", + timezone_name="UTC", + request_timeout_seconds=300, + reconciliation_page_limit=5, + ) + client = DolphinSchedulerClient(scheduler_profile) + binding = client.create_workflow(definition_bundle.workflow) + authorized_at = datetime.now(UTC) + authorization = build_terminal_authorization( + checked_source, + definition_bundle, + binding, + authorized_at=authorized_at, + ) + authorization_sha = _authorization_fingerprint(authorization) + register_terminal_authorization(gateway, authorization) + server.executor = RetainedRealFeatureExecutor( + request, + profile, + plan, + source, + runtime, + rehearsal, + endpoint_url=endpoint_url, + object_store_user=object_store_user, + object_store_material=object_store_material, + authorization_sha256=authorization_sha, + ) + server.start() + adapter = DolphinSchedulerAdapter( + scheduler_profile, + gateway=gateway, + client=client, + clock=lambda: authorized_at, + ) + dispatched = adapter.dispatch( + TENANT, + RUN_ID, + binding, + actor_subject=RUNNER, + attempt_no=1, + ) + terminal_instance = delivery._wait_for_terminal_instance( + client, + dispatched.workflow_instance_id, + binding.workflow_definition_code, + timeout_seconds=terminal_timeout_seconds, + ) + reconciled = adapter.reconcile( + TENANT, + RUN_ID, + binding, + actor_subject=RUNNER, + attempt_no=1, + ) + if ( + terminal_instance.state.upper() != "SUCCESS" + or reconciled.provider_state != "SUCCESS" + or reconciled.run.status != RunStatus.RECONCILING + ): + executor_diagnostic = server.executor + raise RetainedTerminalSuccessError( + "DolphinScheduler did not produce a reconcilable success: " + + json.dumps( + { + "terminal_state": terminal_instance.state.upper(), + "reconciled_state": reconciled.provider_state, + "run_status": reconciled.run.status.value, + "callback_count": ( + executor_diagnostic.request_count + if executor_diagnostic is not None + else 0 + ), + "executor_stage": ( + executor_diagnostic.error_stage + if executor_diagnostic is not None + else None + ), + "executor_error_type": ( + executor_diagnostic.error_type + if executor_diagnostic is not None + else None + ), + }, + ensure_ascii=True, + sort_keys=True, + ) + ) + server_stopped = server.stop() + if client is not None: + client.close() + client = None + executor = server.executor + if ( + executor is None + or executor.error_type is not None + or executor.request_count != 1 + or executor.store is None + or executor.output_contracts is None + or executor.materialized_at is None + or not executor.source_input_removed + ): + raise RetainedTerminalSuccessError( + "retained real feature executor outcome is incomplete" + ) + independent = independently_evaluate_retained_parquet( + runtime, + profile, + plan, + source, + executor.store, + endpoint_url=endpoint_url, + object_store_user=object_store_user, + object_store_material=object_store_material, + ) + latest = _mapping(executor.store.get("latest_metadata")) + data_objects = [ + item + for item in executor.store.get("objects") or [] + if str(_mapping(item).get("key") or "").endswith(".parquet") + ] + retention = build_retained_material_observation( + tenant_id=TENANT, + run_id=RUN_ID, + output_resource_version_id=OUTPUT_RESOURCE_VERSION_ID, + output_content_sha256=plan.output_content_sha256, + storage_uri=profile.target.table_location, + retention_id=retention_id, + owner="team:metadata-platform", + namespace=str(namespace_retention["name"]), + namespace_uid=str(namespace_retention["uid"]), + control_database_ref=control.database_ref, + object_inventory_sha256=str( + executor.store.get("object_inventory_sha256") + ), + metadata_body_sha256=str(latest.get("body_sha256")), + row_set_sha256=plan.row_set_sha256, + snapshot_id=int(latest["current_snapshot_id"]), + feature_count=20, + data_file_count=len(data_objects), + data_size_bytes=int(independent["data_size_bytes"]), + readable=True, + source_payload_retained=False, + materialized_at=executor.materialized_at, + observed_at=datetime.now(UTC), + expires_at=expires_at, + ) + promotion = build_terminal_promotion(checked_source, retention) + def live_probe(observed: RetainedMaterialObservation) -> bool: + return _live_material_probe( + observed, + runtime=runtime, + profile=profile, + control=control, + endpoint_url=endpoint_url, + object_store_user=object_store_user, + object_store_material=object_store_material, + ) + coordinator = RetainedTerminalSuccessCoordinator( + gateway, material_probe=live_probe + ) + first_promotion, succeeded = coordinator.finalize( + promotion, retention, reconciled.observation + ) + replay_promotion, replayed = coordinator.finalize( + promotion, retention, reconciled.observation + ) + counts = _ledger_counts(engine) + control_state = control.observe() + verified = ( + first_promotion.created + and not replay_promotion.created + and succeeded == replayed + and succeeded.status == RunStatus.SUCCEEDED + and succeeded.state_version == 3 + and counts + == { + "artifacts": 5, + "execution_plans": 1, + "policy_decisions": 1, + "approvals": 1, + "evaluator_evidence": 1, + "attempts": 2, + "quality_results": 1, + "lineage_events": 1, + "run_events": 4, + } + and control_state.get("container_running") is True + and control_state.get("volume_retained") is True + and scheduler.cleanup_verified + and server_stopped + and live_probe(retention) + ) + stable = { + "schema": EVIDENCE_SCHEMA, + "status": ( + "local_retained_real_feature_terminal_success_verified" + if verified + else "blocked" + ), + "contract_sha256": contract["contract_sha256"], + "source_ingestion_evidence_sha256": SOURCE_INGESTION_EVIDENCE_SHA256, + "source_promotion_evidence_sha256": SOURCE_PROMOTION_EVIDENCE_SHA256, + "retention_id": retention_id, + "tenant_id": TENANT, + "run_id": str(RUN_ID), + "definition_version_id": str(DEFINITION_VERSION_ID), + "source_resource_version_id": str(SOURCE_RESOURCE_VERSION_ID), + "output_resource_version_id": str(OUTPUT_RESOURCE_VERSION_ID), + "output_content_sha256": plan.output_content_sha256, + "dataset_bundle": source["inventory"], + "source_projection": source["projection"], + "execution_request_sha256": request.request_sha256, + "definition_sha256": definition_bundle.definition.definition_sha256, + "compiled_workflow_sha256": definition_bundle.workflow.compiled_sha256, + "authorization": { + "execution_plan_artifact_id": str(authorization.execution_plan.artifact_id), + "policy_decision_artifact_id": str(authorization.policy_decision.artifact_id), + "approval_artifact_id": str(authorization.approval.artifact_id), + "authorization_sha256": authorization_sha, + "payload_recorded": False, + }, + "provider_observation": { + "observation_id": str(reconciled.observation.observation_id), + "observation_sha256": reconciled.observation.observation_sha256, + "external_namespace": reconciled.observation.external_namespace, + "external_run_id": reconciled.observation.external_run_id, + "observed_state": reconciled.observation.observed_state, + "provider_state": reconciled.provider_state, + }, + "retention_observation": retention.model_dump(mode="json", by_alias=True), + "retention_observation_sha256": retention.observation_sha256, + "namespace_retention": namespace_retention, + "control_database": control_state, + "initial_runtime": initial_runtime, + "object_store_prepared": object_store_prepared, + "bootstrap": bootstrap, + "independent_quality": independent, + "quality_evidence_artifact_id": str( + promotion.quality_evidence_artifact.artifact_id + ), + "quality_result_id": str(promotion.quality_result.quality_result_id), + "lineage_event_id": str(promotion.lineage_event.lineage_event_id), + "output_artifact_id": str(promotion.output_artifact.artifact_id), + "ledger_counts": counts, + "platform_run_status": succeeded.status.value, + "platform_run_state_version": succeeded.state_version, + "scheduler_container_cleanup_verified": scheduler.cleanup_verified, + "execution_callback_cleanup_verified": server_stopped, + "runtime_port_forwards_stopped": False, + "retained_staging_material_verified": verified, + "retained_control_database_verified": verified, + "complete_authorization_artifacts_persisted": verified, + "dolphinscheduler_success_observed": verified, + "independent_quality_evidence_persisted": verified, + "atomic_output_promotion_verified": verified, + "platform_run_succeeded": verified, + "exact_terminal_replay_verified": verified, + "source_payload_removed_from_runtime": verified, + "writes_to_legacy": False, + **{claim: False for claim in FALSE_CLAIMS}, + "errors": [] if verified else ["M3-24 live rehearsal did not verify"], + } + if not verified: + raise RetainedTerminalSuccessError( + "M3-24 live retained rehearsal did not verify" + ) + retained = True + evidence = { + **stable, + "evidence_sha256": canonical_json_fingerprint(stable), + } + finally: + if client is not None: + client.close() + if rehearsal is not None: + rehearsal.close() + if gravitino_forward is not None: + gravitino_forward_stopped = gravitino_forward.stop() + if object_forward is not None: + object_forward_stopped = object_forward.stop() + if server.started and not server.cleanup_verified: + server.stop() + if engine is not None: + engine.dispose() + if not retained: + runtime.cleanup() + control.cleanup() + stable = {key: value for key, value in evidence.items() if key != "evidence_sha256"} + stable["runtime_port_forwards_stopped"] = ( + object_forward_stopped and gravitino_forward_stopped + ) + if not stable["runtime_port_forwards_stopped"]: + raise RetainedTerminalSuccessError("M3-24 runtime port-forward cleanup failed") + return {**stable, "evidence_sha256": canonical_json_fingerprint(stable)} + + +def _apply_migrations(engine: Any) -> None: + m323._apply_migrations(engine) + + +def build_contract_report() -> dict[str, Any]: + errors: list[str] = [] + files = { + "terminal_success": _file_record(Path(__file__).resolve()), + "source_ingestion": _file_record(Path(m322.__file__).resolve()), + "source_promotion": _file_record(Path(m323.__file__).resolve()), + "wrapper": _file_record(DEFAULT_WRAPPER_PATH), + } + try: + source = _load_json_object(DEFAULT_SOURCE_EVIDENCE_PATH) + m323.validate_source_evidence(source) + promotion_evidence = _load_json_object(DEFAULT_PROMOTION_EVIDENCE_PATH) + if m323.validate_rehearsal_evidence(promotion_evidence): + errors.append("M3-23 promotion evidence is invalid") + if promotion_evidence.get("evidence_sha256") != SOURCE_PROMOTION_EVIDENCE_SHA256: + errors.append("M3-23 promotion evidence fingerprint drifted") + except (OSError, ValueError, RetainedTerminalSuccessError): + errors.append("M3-22/M3-23 predecessor evidence is unavailable") + if files["wrapper"]["sha256"] is None: + errors.append("M3-24 wrapper is unavailable") + stable = { + "schema": CONTRACT_SCHEMA, + "source_ingestion_evidence_sha256": SOURCE_INGESTION_EVIDENCE_SHA256, + "source_promotion_evidence_sha256": SOURCE_PROMOTION_EVIDENCE_SHA256, + "files": files, + "requires_retained_material_readback": True, + "requires_complete_authorization_artifacts": True, + "requires_dolphinscheduler_success_observation": True, + "requires_independent_quality_evidence_creator": True, + "uses_existing_atomic_promoter": True, + "uses_existing_success_finalizer": True, + "retained_staging_is_production": False, + "writes_to_legacy": False, + "errors": errors, + } + return { + **stable, + "status": "valid" if not errors else "invalid", + "contract_sha256": canonical_json_fingerprint(stable), + **{claim: False for claim in FALSE_CLAIMS}, + } + + +def validate_evidence(evidence: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + stable = {key: value for key, value in evidence.items() if key != "evidence_sha256"} + if evidence.get("schema") != EVIDENCE_SCHEMA: + errors.append("M3-24 evidence schema does not match") + if evidence.get("evidence_sha256") != canonical_json_fingerprint(stable): + errors.append("M3-24 evidence fingerprint does not match") + contract = build_contract_report() + if evidence.get("contract_sha256") != contract.get("contract_sha256"): + errors.append("M3-24 contract binding is stale") + for claim in ( + "retained_staging_material_verified", + "retained_control_database_verified", + "complete_authorization_artifacts_persisted", + "dolphinscheduler_success_observed", + "independent_quality_evidence_persisted", + "atomic_output_promotion_verified", + "platform_run_succeeded", + "exact_terminal_replay_verified", + "source_payload_removed_from_runtime", + ): + if evidence.get(claim) is not True: + errors.append(f"M3-24 evidence claim is false: {claim}") + for claim in FALSE_CLAIMS: + if evidence.get(claim) is not False: + errors.append(f"M3-24 evidence may not claim {claim}") + try: + retention = RetainedMaterialObservation.model_validate( + evidence.get("retention_observation") + ) + if retention.observation_sha256 != evidence.get( + "retention_observation_sha256" + ): + errors.append("M3-24 retention observation binding drifted") + except ValueError: + errors.append("M3-24 retention observation is invalid") + serialized = json.dumps(evidence, ensure_ascii=True, sort_keys=True) + for forbidden in ( + "/Users/", + "/home/", + "Downloads/", + ".tmp/", + "geometry_wkb_hex", + '"rows"', + '"password"', + '"secret"', + '"token"', + '"access_key"', + '"access-key"', + ): + if forbidden in serialized: + errors.append("M3-24 evidence contains source or secret material") + break + return errors + + +def build_validation_report( + *, evidence_path: Path = DEFAULT_EVIDENCE_PATH +) -> dict[str, Any]: + contract = build_contract_report() + errors = list(contract["errors"]) + evidence: dict[str, Any] | None = None + try: + evidence = _load_json_object(evidence_path) + errors.extend(validate_evidence(evidence)) + except (OSError, ValueError, RetainedTerminalSuccessError): + errors.append("M3-24 checked evidence is unavailable") + return { + "schema": VALIDATION_SCHEMA, + "status": "valid" if not errors else "invalid", + "contract_sha256": contract["contract_sha256"], + "evidence_sha256": evidence.get("evidence_sha256") if evidence else None, + "errors": errors, + } + + +def _write_json(path: Path, payload: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def cleanup_retained_rehearsal( + evidence_path: Path, *, retention_id: str +) -> dict[str, Any]: + evidence = _load_json_object(evidence_path) + if validate_evidence(evidence): + raise RetainedTerminalSuccessError( + "cleanup requires intact checked M3-24 evidence" + ) + if evidence.get("retention_id") != retention_id: + raise RetainedTerminalSuccessError( + "cleanup retention ID does not match checked evidence" + ) + retention = RetainedMaterialObservation.model_validate( + evidence.get("retention_observation") + ) + control = _mapping(evidence.get("control_database")) + namespace_json = _run_command( + ["kubectl", "get", "namespace", retention.namespace, "-o", "json"] + ) + namespace = json.loads(namespace_json) + metadata = _mapping(namespace.get("metadata")) + labels = _mapping(metadata.get("labels")) + if ( + metadata.get("uid") != retention.namespace_uid + or labels.get("gda.gisdataagent.io/retention-id") != retention_id + ): + raise RetainedTerminalSuccessError( + "cleanup namespace ownership does not match checked evidence" + ) + container_name = str(control.get("container_name") or "") + volume_name = str(control.get("volume_name") or "") + if not container_name.startswith("gda-m3-24-control-") or not volume_name.startswith( + "gda-m3-24-control-" + ): + raise RetainedTerminalSuccessError( + "cleanup control database identity is not bounded" + ) + container_retention = _run_command( + [ + "docker", + "container", + "inspect", + container_name, + "--format", + "{{index .Config.Labels \"gda.retention-id\"}}", + ] + ) + volume_retention = _run_command( + [ + "docker", + "volume", + "inspect", + volume_name, + "--format", + "{{index .Labels \"gda.retention-id\"}}", + ] + ) + if container_retention != retention_id or volume_retention != retention_id: + raise RetainedTerminalSuccessError( + "cleanup control database ownership does not match checked evidence" + ) + _run_command( + [ + "kubectl", + "delete", + "namespace", + retention.namespace, + "--wait=true", + "--timeout=5m", + ], + timeout=330, + ) + _run_command(["docker", "rm", "--force", container_name]) + _run_command(["docker", "volume", "rm", volume_name]) + return { + "status": "cleaned", + "retention_id": retention_id, + "namespace_removed": True, + "control_database_removed": True, + "recoverable": False, + } + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + subparsers.add_parser("contract") + validate = subparsers.add_parser("validate") + validate.add_argument("--evidence", type=Path, default=DEFAULT_EVIDENCE_PATH) + live = subparsers.add_parser("live-rehearsal") + live.add_argument("--profile", type=Path, default=m322.DEFAULT_PROFILE_PATH) + live.add_argument("--shapefile", type=Path, required=True) + live.add_argument("--ogrinfo", type=Path, required=True) + live.add_argument("--proj-data", type=Path) + live.add_argument("--output", type=Path, default=DEFAULT_EVIDENCE_PATH) + live.add_argument( + "--scheduler-admin-password-env", + default="GDA_M324_DOLPHINSCHEDULER_ADMIN_PASSWORD", + ) + live.add_argument("--scheduler-readiness-timeout", type=float, default=240) + live.add_argument("--terminal-timeout", type=float, default=1500) + cleanup = subparsers.add_parser("cleanup") + cleanup.add_argument("--evidence", type=Path, default=DEFAULT_EVIDENCE_PATH) + cleanup.add_argument("--retention-id", required=True) + args = parser.parse_args(argv) + if args.command == "contract": + report = build_contract_report() + elif args.command == "validate": + report = build_validation_report(evidence_path=args.evidence) + elif args.command == "live-rehearsal": + report = run_live_rehearsal( + profile_path=args.profile, + shapefile_path=args.shapefile, + ogrinfo_path=args.ogrinfo, + proj_data_path=args.proj_data, + scheduler_admin_password=delivery._read_admin_password( + args.scheduler_admin_password_env + ), + scheduler_readiness_timeout_seconds=args.scheduler_readiness_timeout, + terminal_timeout_seconds=args.terminal_timeout, + ) + errors = validate_evidence(report) + if errors: + raise RetainedTerminalSuccessError( + "M3-24 live evidence failed self-validation: " + "; ".join(errors) + ) + _write_json(args.output, report) + else: + report = cleanup_retained_rehearsal( + args.evidence, retention_id=args.retention_id + ) + print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if report["status"] in { + "valid", + "local_retained_real_feature_terminal_success_verified", + "cleaned", + } else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/data_agent/platform_truth.py b/data_agent/platform_truth.py index 70d19f5b..2cc729b8 100644 --- a/data_agent/platform_truth.py +++ b/data_agent/platform_truth.py @@ -1038,6 +1038,32 @@ def _config( ), "Retained output material, terminal success evidence and production promotion", ), + RuntimeSpec( + "metadata_retained_real_feature_terminal_success_rehearsal", + "retained_real_feature_terminal_success_rehearsal", + "governed", + "retained_evidence_durable", + ( + "retained namespace/PVC/Iceberg material and dedicated GDA Control " + "PostgreSQL until bounded expiry" + ), + "metadata-platform", + "local_verification_only", + ( + "data_agent/metadata_fabric_retained_real_feature_terminal_success.py", + "scripts/metadata-fabric-retained-real-feature-terminal-success.sh", + ), + ( + ( + "data_agent/metadata_fabric_retained_real_feature_terminal_success.py", + "def run_live_rehearsal", + ), + ), + ( + "Protected production identities/storage/tenant binding, persistent " + "scheduler/executor and restart/recovery" + ), + ), RuntimeSpec( "datalake_monitor", "monitor_loop", @@ -1102,7 +1128,7 @@ def _config( "5ee717911c109b480328a050893296e37591bfca748e3ed1743b7e3def3d9048" ) RUNTIME_PRIMITIVE_BASELINE_FINGERPRINT = ( - "499c531821c2138977724d8a9c12e0328cb45525944634c0cfba3763137652b6" + "eecb6bda87e47f3056fbe5772907bba7e6508215825e993d53c735ca5cfa9298" ) _IGNORED_SOURCE_PARTS = frozenset( diff --git a/data_agent/test_metadata_fabric_retained_real_feature_terminal_success.py b/data_agent/test_metadata_fabric_retained_real_feature_terminal_success.py new file mode 100644 index 00000000..4a04687d --- /dev/null +++ b/data_agent/test_metadata_fabric_retained_real_feature_terminal_success.py @@ -0,0 +1,175 @@ +import json +from copy import deepcopy +from datetime import UTC, datetime, timedelta + +import pytest +from pydantic import ValidationError + +from data_agent import metadata_fabric_real_feature_ingestion as m322 +from data_agent import metadata_fabric_retained_real_feature_terminal_success as terminal +from data_agent.dolphinscheduler_adapter import DolphinSchedulerDefinitionBinding + + +def _source() -> dict: + return json.loads(terminal.DEFAULT_SOURCE_EVIDENCE_PATH.read_text(encoding="utf-8")) + + +def _plan() -> m322.RealFeatureIngestionPlan: + return m322.RealFeatureIngestionPlan.model_validate( + _source()["observation"]["plan"] + ) + + +def _request(retention_id: str = "m3-24-unit-retention"): + return terminal.build_execution_request(_plan(), retention_id=retention_id) + + +def _definition(now: datetime): + return terminal.build_terminal_definition( + "http://host.docker.internal:42424/execute", + _request(), + created_at=now, + ) + + +def _binding(now: datetime): + definition = _definition(now) + return definition, DolphinSchedulerDefinitionBinding( + tenant_id=terminal.TENANT, + definition_version_id=terminal.DEFINITION_VERSION_ID, + project_code=2401, + workflow_definition_code=2402, + workflow_definition_version=1, + compiled_sha256=definition.workflow.compiled_sha256, + ) + + +def _retention(now: datetime): + base = terminal.m323.build_promotion(_source()) + return terminal.build_retained_material_observation( + tenant_id=terminal.TENANT, + run_id=terminal.RUN_ID, + output_resource_version_id=terminal.OUTPUT_RESOURCE_VERSION_ID, + output_content_sha256=base.output_resource_version.content_sha256, + storage_uri=base.output_artifact.storage_uri, + retention_id="m3-24-unit-retention", + owner="team:metadata-platform", + namespace="gda-metadata-spark-object-store", + namespace_uid="00000000-0000-4000-8000-000000000024", + control_database_ref="docker:gda-m3-24-control-unit", + object_inventory_sha256="1" * 64, + metadata_body_sha256="2" * 64, + row_set_sha256=base.output_artifact.manifest["row_set_sha256"], + snapshot_id=base.output_artifact.manifest["snapshot_id"], + feature_count=20, + data_file_count=1, + data_size_bytes=base.output_artifact.size_bytes, + readable=True, + source_payload_retained=False, + materialized_at=now, + observed_at=now + timedelta(seconds=2), + expires_at=now + timedelta(days=7), + ) + + +def test_execution_request_and_definition_bind_exact_real_feature_plan(): + now = datetime(2026, 7, 31, 2, tzinfo=UTC) + request = _request() + definition = _definition(now) + + assert request.ingestion_plan_sha256 == _plan().ingestion_plan_sha256 + assert request.output_content_sha256 == _plan().output_content_sha256 + assert definition.definition.output_contract["retained_staging_material"] is True + assert definition.definition.output_contract["platform_run_terminal_success"] is True + raw_script = definition.workflow.task_definitions[0]["taskParams"]["rawScript"] + assert "curl --fail" in raw_script + assert request.request_sha256 in raw_script + + +def test_execution_request_survives_real_json_transport(): + request = _request() + transported = json.loads(request.model_dump_json(by_alias=True)) + + assert terminal.RetainedExecutionRequest.model_validate(transported) == request + + +def test_complete_authorization_uses_dolphinscheduler_dispatch_and_approval(): + now = datetime(2026, 7, 31, 2, tzinfo=UTC) + definition, binding = _binding(now) + bundle = terminal.build_terminal_authorization( + _source(), definition, binding, authorized_at=now + timedelta(minutes=1) + ) + + assert bundle.run.policy_refs is not None + assert bundle.run.policy_refs.policy_decision_artifact_id == ( + bundle.policy_decision.artifact_id + ) + assert bundle.run.policy_refs.approval_artifact_id == bundle.approval.artifact_id + assert bundle.execution_plan.artifact_role.value == "execution_plan" + decision = bundle.policy_decision.manifest["decision"] + assert decision["action"] == "dolphinscheduler.dispatch" + assert decision["subject_context"]["subject_id"] == ( + terminal.RUNNER.removeprefix("workload:") + ) + + +def test_retained_promotion_replaces_executor_quality_evidence_provenance(): + now = datetime(2026, 7, 31, 3, tzinfo=UTC) + retention = _retention(now) + promotion = terminal.build_terminal_promotion(_source(), retention) + + assert promotion.output_artifact.storage_uri == retention.storage_uri + assert promotion.output_artifact.manifest["retention_id"] == retention.retention_id + assert promotion.quality_evidence_artifact.created_by == terminal.QUALITY_EVALUATOR + assert promotion.quality_result.evaluated_by == terminal.QUALITY_EVALUATOR + assert promotion.quality_evidence_artifact.created_by == ( + promotion.quality_result.evaluated_by + ) + assert promotion.quality_result.metrics["independent_material_readback"] is True + assert promotion.lineage_event.facets["retention_id"] == retention.retention_id + + +def test_retained_observation_rejects_unordered_expiry(): + now = datetime(2026, 7, 31, 3, tzinfo=UTC) + values = _retention(now).model_dump(mode="python", by_alias=True) + values["expires_at"] = now + timedelta(seconds=1) + values.pop("observation_sha256") + + with pytest.raises(ValidationError, match="timestamps are not ordered"): + terminal.build_retained_material_observation(**values) + + +def test_retained_promotion_rejects_different_material_identity(): + now = datetime(2026, 7, 31, 3, tzinfo=UTC) + values = _retention(now).model_dump(mode="python", by_alias=True) + values.pop("observation_sha256") + values["row_set_sha256"] = "0" * 64 + retention = terminal.build_retained_material_observation(**values) + + with pytest.raises( + terminal.RetainedTerminalSuccessError, + match="does not bind", + ): + terminal.build_terminal_promotion(_source(), retention) + + +def test_checked_predecessor_tampering_is_rejected(): + source = deepcopy(_source()) + source["observation"]["plan"]["output_content_sha256"] = "0" * 64 + now = datetime(2026, 7, 31, 3, tzinfo=UTC) + + with pytest.raises(terminal.m323.RealFeatureLedgerPromotionError): + terminal.build_terminal_promotion(source, _retention(now)) + + +def test_contract_keeps_retained_staging_below_production_boundary(): + report = terminal.build_contract_report() + + assert report["status"] == "valid" + assert report["errors"] == [] + assert report["requires_retained_material_readback"] is True + assert report["requires_complete_authorization_artifacts"] is True + assert report["requires_dolphinscheduler_success_observation"] is True + assert report["requires_independent_quality_evidence_creator"] is True + assert report["retained_staging_is_production"] is False + assert report["production_ready"] is False diff --git a/data_agent/test_metadata_fabric_retained_real_feature_terminal_success_postgres.py b/data_agent/test_metadata_fabric_retained_real_feature_terminal_success_postgres.py new file mode 100644 index 00000000..75afcc61 --- /dev/null +++ b/data_agent/test_metadata_fabric_retained_real_feature_terminal_success_postgres.py @@ -0,0 +1,360 @@ +import json +import os +from datetime import UTC, datetime, timedelta +from uuid import uuid4, uuid5 + +import pytest +from sqlalchemy import create_engine, text +from sqlalchemy.engine import make_url + +from data_agent import metadata_fabric_retained_real_feature_terminal_success as terminal +from data_agent.dolphinscheduler_adapter import DolphinSchedulerDefinitionBinding +from data_agent.platform_contracts import ( + FrameworkAttemptObservation, + canonical_json_fingerprint, +) +from data_agent.platform_gateway import ( + GatewayConflictError, + GatewayValidationError, + PlatformGateway, +) + +DATABASE_URL = os.environ.get("DATABASE_URL") + + +def _temporary_database_url(prefix: str) -> tuple[object, str, str]: + admin_url = make_url(DATABASE_URL) + admin_engine = create_engine(admin_url, isolation_level="AUTOCOMMIT") + with admin_engine.connect() as connection: + if not connection.exec_driver_sql( + "SELECT rolsuper FROM pg_roles WHERE rolname = current_user" + ).scalar_one(): + admin_engine.dispose() + pytest.skip("M3-24 PostgreSQL test requires a superuser") + database_name = f"{prefix}_{uuid4().hex}" + connection.exec_driver_sql(f'CREATE DATABASE "{database_name}"') + database_url = admin_url.set(database=database_name).render_as_string( + hide_password=False + ) + return admin_engine, database_name, database_url + + +def _drop_temporary_database(admin_engine, database_name: str) -> None: + with admin_engine.connect() as connection: + connection.execute( + text( + """ + SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname = :database_name + AND pid <> pg_backend_pid() + """ + ), + {"database_name": database_name}, + ) + connection.exec_driver_sql(f'DROP DATABASE "{database_name}"') + admin_engine.dispose() + + +def _source() -> dict: + return json.loads(terminal.DEFAULT_SOURCE_EVIDENCE_PATH.read_text(encoding="utf-8")) + + +def _fixtures(now: datetime): + source = _source() + plan = terminal.m322.RealFeatureIngestionPlan.model_validate( + source["observation"]["plan"] + ) + request = terminal.build_execution_request( + plan, retention_id="m3-24-postgres-retention" + ) + definition = terminal.build_terminal_definition( + "http://host.docker.internal:42424/execute", + request, + created_at=now, + ) + binding = DolphinSchedulerDefinitionBinding( + tenant_id=terminal.TENANT, + definition_version_id=terminal.DEFINITION_VERSION_ID, + project_code=2401, + workflow_definition_code=2402, + workflow_definition_version=1, + compiled_sha256=definition.workflow.compiled_sha256, + ) + authorization = terminal.build_terminal_authorization( + source, + definition, + binding, + authorized_at=now + timedelta(minutes=1), + ) + base = terminal.m323.build_promotion(source) + retention = terminal.build_retained_material_observation( + tenant_id=terminal.TENANT, + run_id=terminal.RUN_ID, + output_resource_version_id=terminal.OUTPUT_RESOURCE_VERSION_ID, + output_content_sha256=base.output_resource_version.content_sha256, + storage_uri=base.output_artifact.storage_uri, + retention_id="m3-24-postgres-retention", + owner="team:metadata-platform", + namespace="gda-metadata-spark-object-store", + namespace_uid="00000000-0000-4000-8000-000000000024", + control_database_ref="postgresql:gda-m3-24-postgres-test", + object_inventory_sha256="1" * 64, + metadata_body_sha256="2" * 64, + row_set_sha256=base.output_artifact.manifest["row_set_sha256"], + snapshot_id=base.output_artifact.manifest["snapshot_id"], + feature_count=20, + data_file_count=1, + data_size_bytes=base.output_artifact.size_bytes, + readable=True, + source_payload_retained=False, + materialized_at=now + timedelta(minutes=2), + observed_at=now + timedelta(minutes=3), + expires_at=now + timedelta(days=7), + ) + promotion = terminal.build_terminal_promotion(source, retention) + provider_evidence = { + "api_profile": "3.4", + "project_code": binding.project_code, + "server_version": "3.4.2", + "workflow_definition_code": binding.workflow_definition_code, + "workflow_definition_version": binding.workflow_definition_version, + "workflow_instance_id": 2403, + "provider_state": "SUCCESS", + "provider_start_time": (now + timedelta(minutes=1)).isoformat(), + "provider_end_time": (now + timedelta(minutes=2)).isoformat(), + } + observation = FrameworkAttemptObservation( + tenant_id=terminal.TENANT, + observation_id=uuid5(terminal.RUN_ID, "m3-24-provider-success"), + run_id=terminal.RUN_ID, + attempt_no=1, + framework_kind="dolphinscheduler", + external_namespace=str(binding.project_code), + external_run_id="2403", + external_attempt_id=None, + observed_state="success", + observation_sha256=canonical_json_fingerprint(provider_evidence), + evidence=provider_evidence, + observed_at=now + timedelta(minutes=2), + ) + return authorization, retention, promotion, observation + + +def _register_reconciling_run(gateway, authorization, observation): + terminal.register_terminal_authorization(gateway, authorization) + gateway.transition_run( + terminal.TENANT, + terminal.RUN_ID, + 0, + "dispatching", + terminal.RUNNER, + "dispatching retained real-feature ingestion", + ) + gateway.record_attempt(observation) + return gateway.transition_run( + terminal.TENANT, + terminal.RUN_ID, + 1, + "reconciling", + terminal.RUNNER, + "DolphinScheduler reached terminal provider state", + ) + + +@pytest.mark.skipif(not DATABASE_URL, reason="DATABASE_URL is not configured") +def test_retained_terminal_evidence_succeeds_and_exactly_replays(): + admin_engine, database_name, database_url = _temporary_database_url( + "gda_retained_terminal_success" + ) + engine = None + try: + now = datetime(2026, 7, 31, 4, tzinfo=UTC) + authorization, retention, promotion, observation = _fixtures(now) + engine = create_engine(database_url) + terminal._apply_migrations(engine) + gateway = PlatformGateway(engine) + _register_reconciling_run(gateway, authorization, observation) + coordinator = terminal.RetainedTerminalSuccessCoordinator( + gateway, material_probe=lambda observed: observed == retention + ) + + first_promotion, first_run = coordinator.finalize( + promotion, retention, observation + ) + replay_promotion, replay_run = coordinator.finalize( + promotion, retention, observation + ) + + assert first_promotion.created is True + assert replay_promotion.created is False + assert first_run == replay_run + assert first_run.status.value == "succeeded" + assert first_run.state_version == 3 + with engine.connect() as connection: + row = connection.execute( + text( + """ + SELECT + count(*) FILTER (WHERE artifact_role = 'execution_plan') + AS execution_plans, + count(*) FILTER ( + WHERE media_type = 'application/vnd.gda.policy-decision+json' + ) AS policy_decisions, + count(*) FILTER ( + WHERE media_type = 'application/vnd.gda.approval+json' + ) AS approvals, + count(*) FILTER ( + WHERE created_by = :quality_evaluator + AND run_id = :run_id + ) AS evaluator_evidence + FROM gda_control.artifact + WHERE tenant_id = :tenant_id + """ + ), + { + "tenant_id": terminal.TENANT, + "run_id": terminal.RUN_ID, + "quality_evaluator": terminal.QUALITY_EVALUATOR, + }, + ).one() + assert tuple(row) == (1, 1, 1, 1) + finally: + if engine is not None: + engine.dispose() + _drop_temporary_database(admin_engine, database_name) + + +@pytest.mark.skipif(not DATABASE_URL, reason="DATABASE_URL is not configured") +def test_unreadable_material_rejects_before_output_promotion(): + admin_engine, database_name, database_url = _temporary_database_url( + "gda_unreadable_retained_material" + ) + engine = None + try: + now = datetime(2026, 7, 31, 4, tzinfo=UTC) + authorization, retention, promotion, observation = _fixtures(now) + engine = create_engine(database_url) + terminal._apply_migrations(engine) + gateway = PlatformGateway(engine) + _register_reconciling_run(gateway, authorization, observation) + coordinator = terminal.RetainedTerminalSuccessCoordinator( + gateway, material_probe=lambda _observed: False + ) + + with pytest.raises( + terminal.RetainedTerminalSuccessError, + match="absent, expired, or unreadable", + ): + coordinator.finalize(promotion, retention, observation) + + with engine.connect() as connection: + counts = connection.execute( + text( + """ + SELECT + (SELECT count(*) FROM gda_control.quality_result) AS quality, + (SELECT count(*) FROM gda_control.lineage_event) AS lineage, + (SELECT count(*) FROM gda_control.resource_version + WHERE resource_version_id = :output_id) AS output_versions + """ + ), + {"output_id": terminal.OUTPUT_RESOURCE_VERSION_ID}, + ).one() + assert tuple(counts) == (0, 0, 0) + assert gateway.get_run(terminal.TENANT, terminal.RUN_ID).status.value == ( + "reconciling" + ) + finally: + if engine is not None: + engine.dispose() + _drop_temporary_database(admin_engine, database_name) + + +@pytest.mark.skipif(not DATABASE_URL, reason="DATABASE_URL is not configured") +def test_quality_creator_impersonation_and_wrong_provider_state_fail_closed(): + admin_engine, database_name, database_url = _temporary_database_url( + "gda_retained_provenance_negative" + ) + engine = None + try: + now = datetime(2026, 7, 31, 4, tzinfo=UTC) + authorization, retention, promotion, observation = _fixtures(now) + engine = create_engine(database_url) + terminal._apply_migrations(engine) + gateway = PlatformGateway(engine) + _register_reconciling_run(gateway, authorization, observation) + coordinator = terminal.RetainedTerminalSuccessCoordinator( + gateway, material_probe=lambda _observed: True + ) + + wrong_observation = observation.model_copy(update={"observed_state": "failed"}) + with pytest.raises( + terminal.RetainedTerminalSuccessError, + match="DolphinScheduler success", + ): + coordinator.finalize(promotion, retention, wrong_observation) + + values = promotion.model_dump(mode="python") + values["quality_evidence_artifact"] = ( + promotion.quality_evidence_artifact.model_copy( + update={"created_by": terminal.RUNNER} + ) + ) + impersonated = terminal.m323.RunOutputLedgerPromotion.model_validate(values) + with pytest.raises( + terminal.RetainedTerminalSuccessError, + match="independent evaluator", + ): + coordinator.finalize(impersonated, retention, observation) + assert gateway.get_run(terminal.TENANT, terminal.RUN_ID).status.value == ( + "reconciling" + ) + finally: + if engine is not None: + engine.dispose() + _drop_temporary_database(admin_engine, database_name) + + +@pytest.mark.skipif(not DATABASE_URL, reason="DATABASE_URL is not configured") +def test_missing_policy_artifact_and_conflicting_terminal_reason_are_rejected(): + admin_engine, database_name, database_url = _temporary_database_url( + "gda_retained_authority_negative" + ) + engine = None + try: + now = datetime(2026, 7, 31, 4, tzinfo=UTC) + authorization, retention, promotion, observation = _fixtures(now) + engine = create_engine(database_url) + terminal._apply_migrations(engine) + gateway = PlatformGateway(engine) + gateway.register_resource(authorization.source_resource) + gateway.register_resource_version(authorization.source_version) + gateway.register_definition(authorization.definition_registration) + gateway.register_resource(authorization.output_resource) + gateway.record_artifact(authorization.execution_plan) + gateway.record_artifact(authorization.approval) + with pytest.raises(GatewayValidationError, match="Policy decision artifact"): + gateway.submit_run(authorization.run) + + gateway.record_artifact(authorization.policy_decision) + _register_reconciling_run(gateway, authorization, observation) + coordinator = terminal.RetainedTerminalSuccessCoordinator( + gateway, material_probe=lambda _observed: True + ) + _promoted, succeeded = coordinator.finalize( + promotion, retention, observation + ) + assert succeeded.status.value == "succeeded" + + with pytest.raises(GatewayConflictError, match="platform state conflict"): + coordinator.finalize( + promotion, + retention, + observation, + reason="conflicting terminal reason", + ) + finally: + if engine is not None: + engine.dispose() + _drop_temporary_database(admin_engine, database_name) diff --git a/data_agent/test_platform_truth.py b/data_agent/test_platform_truth.py index 5f90b932..690b5736 100644 --- a/data_agent/test_platform_truth.py +++ b/data_agent/test_platform_truth.py @@ -303,6 +303,13 @@ def test_repository_source_access_and_runtime_baselines_match(): and item["production_role"] == "local_verification_only" for item in static_report["runtime"]["inventory"] ) + assert any( + item["runtime_id"] + == "metadata_retained_real_feature_terminal_success_rehearsal" + and item["durability"] == "retained_evidence_durable" + and item["production_role"] == "local_verification_only" + for item in static_report["runtime"]["inventory"] + ) def test_runtime_report_detects_unregistered_background_mechanism(tmp_path): diff --git a/docs/architecture-decisions/adr-070-retained-real-feature-terminal-success.md b/docs/architecture-decisions/adr-070-retained-real-feature-terminal-success.md new file mode 100644 index 00000000..325e8a51 --- /dev/null +++ b/docs/architecture-decisions/adr-070-retained-real-feature-terminal-success.md @@ -0,0 +1,80 @@ +# ADR-070: Retained real-feature terminal success rehearsal + +**Status**: Accepted + +**Date**: 2026-07-31 + +**Decision owners**: Data Platform, Metadata Platform, Data Governance, GIS Platform, Security, Platform Architecture + +**Related decisions**: [ADR-007](adr-007-dolphinscheduler-temporal-orchestration-platform.md) · [ADR-026](adr-026-evidence-gated-run-success.md) · [ADR-068](adr-068-local-authorized-real-feature-iceberg-ingestion.md) · [ADR-069](adr-069-atomic-real-feature-output-ledger-promotion.md) + +## Context + +M3-22 proved real Spark/Sedona ingestion into a temporary JDBC/S3 Iceberg runtime. M3-23 proved that its path-free output, quality and lineage candidates could be promoted atomically into a temporary GDA Control ledger. Both rehearsals deliberately deleted their material and left the correlated PlatformRun non-terminal because complete authorization Artifacts, a provider success observation and independently created quality evidence were not all present together. + +The next gate must prove the complete chain against one retained output without weakening the existing authority boundaries. A scheduler state alone cannot finalize a Run, a ledger promotion alone cannot make missing material readable, and retained local infrastructure cannot be described as production. + +## Decision + +### 1. Execute the checked real-data plan through DolphinScheduler + +M3-24 uses the same content-bound Chongqing 20-feature EPSG:4490 cultural-district slice as M3-22. A DolphinScheduler `3.4.2` Shell task calls a bounded ephemeral executor, and that executor runs the checked Spark `3.5.0` + Sedona `1.9.0` JDBC/S3 Iceberg ingestion plan. + +The execution plan, PolicyDecision and Approval Artifacts are written to GDA Control before dispatch. The provider callback is accepted only when it binds the exact tenant, Run, definition, compiled workflow and request. The resulting FrameworkAttemptObservation must contain a real DolphinScheduler `SUCCESS`; it remains an observation rather than the PlatformRun authority. + +### 2. Retain material long enough to audit terminal success + +After successful ingestion, the source ConfigMap containing identifiers and WKB payload is deleted. The namespace UID, PVC-backed catalog state, MinIO/Iceberg output and dedicated GDA Control PostgreSQL database are retained for seven days under one `retention_id` and one expiry timestamp. + +Committed evidence contains aggregate inventory, hashes, counts and bounded runtime identities. It does not contain the source absolute path, feature identifiers, geometry bytes, credentials or the source payload. `retained_staging_material_verified=true` means the recorded output was readable during the rehearsal and remains available for bounded audit until expiry; it does not establish production durability. + +### 3. Re-evaluate the retained Parquet independently + +The quality evaluator reopens the single retained Parquet object independently of the ingestion executor. It recomputes the canonical row-set fingerprint and verifies feature count, unique identifiers, non-empty/valid Z geometry, SRID, positive area and source-matching bounds. + +The evaluator creates its own quality evidence Artifact. That Artifact creator must equal the QualityResult evaluator and must differ from the output creator. An executor-authored substitute fails the existing success gate. + +### 4. Keep output promotion and terminal authority separate + +M3-24 reuses the M3-23 atomic promoter without modifying the evidence-bound PlatformGateway. The output ResourceVersion, output Artifact, evaluator evidence Artifact, passed QualityResult and source-to-output LineageEvent are appended as one exact-replay bundle. + +Only after retained material readback, complete authorization, the exact provider success observation, independent quality provenance and lineage all agree may migration 096's existing database finalizer move the Run from `reconciling@2` to `succeeded@3`. Exact terminal replay must create no new output facts, attempt observation or Run event. + +### 5. Make cleanup explicit and identity-bound + +Cleanup is a separate operator action. It requires intact checked evidence and an exact `retention_id`; it verifies namespace UID/label and control container/volume labels before deleting anything. The recorded M3-24 cleanup invocation is: + +```bash +./scripts/metadata-fabric-retained-real-feature-terminal-success.sh \ + cleanup --retention-id m3-24-229740ac50ebb53b +``` + +The retained resources must not be cleaned before their audit window is complete unless an authorized operator deliberately invokes that bounded command. This ADR records the lifecycle contract; it does not authorize early cleanup. + +### 6. Cap claims below production + +M3-24 is a retained local staging rehearsal. Its scheduler is temporary, its executor callback is ephemeral, its catalog/object store and GDA Control database run on one development host, and its identities are not protected production workload identities. It does not prove production scheduler HA, independent storage failure domains, KMS/TLS/OIDC, tenant attestation, backup/PITR, restart recovery, staging scale or production ingestion. + +Therefore `production_scheduler_verified`, `protected_workload_identity_verified`, `production_object_store_verified`, `production_tenant_attestation_verified`, `production_ingestion_verified` and `production_ready` remain false. + +## Verification + +The M3-24 rehearsal recorded: + +- contract SHA `9c8f20ca1fb9995530c4e988ced627f665857ecdf0e104bb7d07c4a4a486057a` and evidence SHA `d966668b5a2ea57c7a4b2a3bc9824daab9b0128d9f94e515d7be649b145de418`; +- retention ID `m3-24-229740ac50ebb53b`, namespace UID `824a5904-70cc-4a85-8503-ca83acbcde16` and expiry `2026-08-07T04:15:23.082316Z`; +- output content SHA `bdc06792e8b935176ee6df6f6f6d4be1535622d54d9b994a778cabfe5a574618`, row-set SHA `c26ff708f4b6be082327dff63a6a8659420dbc4cab37dea1cac7b40f147512df` and one 94,603-byte Parquet object; +- nine independent quality counts equal to 20; +- one PolicyDecision, one Approval, one execution plan, five total Artifacts, two attempt observations, one QualityResult and one LineageEvent; +- a real DolphinScheduler `SUCCESS`, final PlatformRun `succeeded@3` and an exact terminal replay with no additional facts; +- deleted source payload ConfigMap and scheduler container, with the namespace/PVC/material and dedicated GDA Control database retained. + +## Consequences + +**Positive**: the platform now has one auditable real-data path from authorization through scheduler execution, spatial lakehouse material, atomic control-ledger promotion and database-authoritative terminal success. + +**Negative**: retained local resources require an explicit lifecycle and consume workstation capacity for seven days. The result remains below production because the scheduler, executor, identities, storage and control database are not production deployments. + +**Next gate**: repeat the same authority chain with protected production identities and tenant binding, selected production catalog/object storage, persistent scheduler/executor deployments, independent failure domains, backup/PITR and restart/recovery evidence. Then add staging-scale and Spark/Flink conformance before claiming production ingestion. + +**Revisit trigger**: replace the seven-day local retention policy when an approved staging environment supplies durable lifecycle automation, immutable retention policy, ownership, backup and auditable cleanup through the platform control plane. diff --git a/docs/evidence/metadata-fabric-retained-real-feature-terminal-success-2026-07-31.json b/docs/evidence/metadata-fabric-retained-real-feature-terminal-success-2026-07-31.json new file mode 100644 index 00000000..240f3769 --- /dev/null +++ b/docs/evidence/metadata-fabric-retained-real-feature-terminal-success-2026-07-31.json @@ -0,0 +1,412 @@ +{ + "atomic_output_promotion_verified": true, + "authorization": { + "approval_artifact_id": "66045674-a229-5c12-81e7-53750c215d63", + "authorization_sha256": "ecfa760b8f607d3b5fe16c851cf632eec227e37e8620d4d82809b3f15c885cd2", + "execution_plan_artifact_id": "6190821e-d05e-5d08-8db8-bc077ac42c42", + "payload_recorded": false, + "policy_decision_artifact_id": "8df5b46d-8bb8-5501-827e-6cea9fe35597" + }, + "bootstrap": { + "admin_authentication_status": 200, + "bounded_authentication_status": 200, + "bucket": "gda-metadata-warehouse", + "catalog": "lakehouse", + "catalog_backend": "jdbc", + "catalog_uri": "jdbc:postgresql://gravitino-persistence-postgresql:5432/iceberg", + "denied_catalog_create_status": 403, + "io_impl": "org.apache.iceberg.aws.s3.S3FileIO", + "material_recorded": false, + "metalake": "gda_chongqing_m3_22", + "role": { + "name": "gda-object-store-cultural-district-projector", + "securable_objects": [ + { + "fullName": "lakehouse", + "privileges": [ + { + "condition": "ALLOW", + "name": "USE_CATALOG" + } + ], + "type": "CATALOG" + }, + { + "fullName": "lakehouse.cultural_heritage", + "privileges": [ + { + "condition": "ALLOW", + "name": "CREATE_TABLE" + }, + { + "condition": "ALLOW", + "name": "USE_SCHEMA" + } + ], + "type": "SCHEMA" + } + ] + }, + "s3_endpoint": "http://metadata-object-store:9000", + "s3_path_style_access": true, + "s3_region": "us-east-1", + "schema": "cultural_heritage", + "server_version": "1.3.0", + "warehouse": "s3://gda-metadata-warehouse/warehouse" + }, + "compiled_workflow_sha256": "fca992829a21d46deaa653eb5ffd5f1072a98ad6111509e3a248c066f6228a9a", + "complete_authorization_artifacts_persisted": true, + "contract_sha256": "9c8f20ca1fb9995530c4e988ced627f665857ecdf0e104bb7d07c4a4a486057a", + "control_database": { + "container_name": "gda-m3-24-control-229740ac50ebb53b", + "container_running": true, + "container_status": "running", + "credential_recorded": false, + "database_ref": "docker:gda-m3-24-control-229740ac50ebb53b/postgres", + "expires_at": "2026-08-07T04:15:23.082316Z", + "host_port": 64667, + "owner": "team:metadata-platform", + "retention_id": "m3-24-229740ac50ebb53b", + "volume_name": "gda-m3-24-control-229740ac50ebb53b", + "volume_retained": true + }, + "dataset_bundle": { + "components": [ + { + "component": ".cpg", + "sha256": "3ad3031f5503a4404af825262ee8232cc04d4ea6683d42c5dd0a2f2a27ac9824", + "size_bytes": 5 + }, + { + "component": ".dbf", + "sha256": "ee7c6c4c6957aea296b69d62118d416e5ee989aa77f7b98cf0fe580874ce5127", + "size_bytes": 44990 + }, + { + "component": ".prj", + "sha256": "b10dbe4d6d1de908d340f892c90b3d31a552630af3742bb515bfe1bd26124f2c", + "size_bytes": 176 + }, + { + "component": ".sbn", + "sha256": "7d0279465b18beec40308717e0ef0ea5701a586bc5c84e6a9aa309d5bc0ec99a", + "size_bytes": 308 + }, + { + "component": ".sbx", + "sha256": "019156149b2c7771ec0dd249c757dd7aa01a98075e246a77f77b080860c57333", + "size_bytes": 124 + }, + { + "component": ".shp", + "sha256": "6ac0d5c8c8db66fc0e2a74d8232b7779bd2454257df14efa2930e3dbc181aed0", + "size_bytes": 283640 + }, + { + "component": ".shp.xml", + "sha256": "8ef222ce1952552b366acf14a996e1c8cbdbe3eed0bb829dacfe7eafd068d948", + "size_bytes": 43100 + }, + { + "component": ".shx", + "sha256": "f3fbb6a7775ca833c066e3a3f2a332f99f979840045909ae187d08dac126a119", + "size_bytes": 260 + } + ], + "content_sha256": "fd474fd65c8e4a71da241eb3fd07748ca3b972fbd2d3c32833376dbe71104007", + "format": "ESRI Shapefile", + "schema": "gda.spatial_dataset_bundle.v1", + "source_label": "chongqing-central-cultural-districts", + "spatial_inventory": { + "bounds": [ + 106.37987914500007, + 29.558008447000077, + 106.59532712300008, + 29.877271985000025 + ], + "crs": { + "authority": "EPSG", + "code": 4490, + "name": "China Geodetic Coordinate System 2000" + }, + "driver": "ESRI Shapefile", + "feature_count": 20, + "field_count": 33, + "geometry_type": "PolygonZ" + } + }, + "definition_sha256": "214fd4b960144ca822de7871c2d4bf3f8da5735f7f455e7a1720e7b4d6532a3b", + "definition_version_id": "a9000000-0000-4000-8000-000000000008", + "dolphinscheduler_success_observed": true, + "durable_catalog_verified": false, + "errors": [], + "evidence_sha256": "d966668b5a2ea57c7a4b2a3bc9824daab9b0128d9f94e515d7be649b145de418", + "exact_terminal_replay_verified": true, + "execution_callback_cleanup_verified": true, + "execution_request_sha256": "1307e7ce78583e9fd0fb2d30b1aa5976bc87d30e17f1b0c5973ce2d8c10b2d7c", + "independent_quality": { + "data_body_sha256": "6cc0fc9eaf48f8106f9afe192704c44407c86c9ea119ae20894bf369a8e74779", + "data_key_sha256": "fbbb249a176bff74f3ee1b756aad1a5233fa007347f68cf466606ddae78e8402", + "data_size_bytes": 94603, + "feature_payload_recorded": false, + "geometry_values_recorded": false, + "identifier_values_recorded": false, + "metrics": { + "bbox_match_count": 20, + "feature_count": 20, + "geometry_z_count": 20, + "non_empty_geometry_count": 20, + "positive_area_count": 20, + "row_fingerprint_match_count": 20, + "srid_match_count": 20, + "unique_bsm_count": 20, + "valid_geometry_count": 20 + }, + "row_set_sha256": "c26ff708f4b6be082327dff63a6a8659420dbc4cab37dea1cac7b40f147512df" + }, + "independent_quality_evidence_persisted": true, + "initial_runtime": { + "context": "docker-desktop", + "gravitino": { + "image": "docker.io/gda/gravitino:1.3.0-local-arm64", + "image_id": "sha256:18e24b43be854dabdc13e96b1019eb3dc691d59cc64e411aa6a3cc49225fe2d3", + "node_name": "desktop-worker", + "persistent_volume_claims": [], + "pod_name": "gravitino-persistence-0", + "pod_uid": "f5e93b4f-e3ab-4fe5-a457-3ad4c7bcb326", + "pvc": null, + "ready_replicas": 1, + "service_account": "gravitino-persistence", + "service_account_automount_disabled": true, + "statefulset_uid": "a316201a-8ba0-4b27-8ce8-bc27965d9b10" + }, + "gravitino_aws_sdk_mounted": true, + "gravitino_host_image_id": "sha256:d355dc7e92f9e3545d717f3eab2cbdf412115f2b82e1e544d7f6235c1eacd5a5", + "gravitino_jdbc_driver_mounted": true, + "iceberg_rest": { + "aws_sdk_mounted": true, + "image": "docker.io/gda/gravitino:1.3.0-local-arm64", + "image_id": "sha256:18e24b43be854dabdc13e96b1019eb3dc691d59cc64e411aa6a3cc49225fe2d3", + "jdbc_driver_mounted": true, + "path": "/iceberg", + "ready": true + }, + "minio_host_image_id": "sha256:a1ea29fa28355559ef137d71fc570e508a214ec84ff8083e39bc5428980b015e", + "namespace": { + "name": "gda-metadata-spark-object-store", + "uid": "824a5904-70cc-4a85-8503-ca83acbcde16" + }, + "object_store": { + "image": "docker.io/minio/minio:RELEASE.2025-04-22T22-12-26Z", + "image_id": "docker.io/minio/minio@sha256:a1ea29fa28355559ef137d71fc570e508a214ec84ff8083e39bc5428980b015e", + "node_name": "desktop-control-plane", + "persistent_volume_claims": [ + "data-metadata-object-store-0" + ], + "pod_name": "metadata-object-store-0", + "pod_uid": "2509ebb5-7ba0-49ea-a6e8-5bffb1a1d4b7", + "pvc": { + "name": "data-metadata-object-store-0", + "phase": "Bound", + "storage_class": "standard", + "uid": "bddccc8e-1da2-4155-9c8c-a8a23a87b7a4", + "volume_name": "pvc-bddccc8e-1da2-4155-9c8c-a8a23a87b7a4" + }, + "ready_replicas": 1, + "service_account": "metadata-object-store", + "service_account_automount_disabled": true, + "statefulset_uid": "d8e3727b-8812-4c6c-9dc3-248f6f1132cd" + }, + "object_store_service": { + "name": "metadata-object-store", + "ports": [ + { + "name": "api", + "port": 9000 + } + ], + "type": "ClusterIP", + "uid": "47b4845f-6776-40f5-8f31-781c5eea460c" + }, + "postgresql": { + "image": "docker.io/library/postgres:16.10-bookworm", + "image_id": "docker.io/library/postgres@sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74", + "node_name": "desktop-worker", + "persistent_volume_claims": [ + "data-gravitino-persistence-postgresql-0" + ], + "pod_name": "gravitino-persistence-postgresql-0", + "pod_uid": "370c3e58-f3bd-4e86-8c75-984b8332ee48", + "pvc": { + "name": "data-gravitino-persistence-postgresql-0", + "phase": "Bound", + "storage_class": "standard", + "uid": "6b4cdb87-5f96-4f65-a571-d8390f6ab405", + "volume_name": "pvc-6b4cdb87-5f96-4f65-a571-d8390f6ab405" + }, + "ready_replicas": 1, + "service_account": "gravitino-persistence-postgresql", + "service_account_automount_disabled": true, + "statefulset_uid": "a2711eb2-a329-4264-8ac8-a6b52b186cbb" + }, + "service": { + "name": "gravitino-persistence", + "ports": [ + { + "name": "http", + "port": 8090 + }, + { + "name": "iceberg-rest", + "port": 9001 + } + ], + "type": "ClusterIP", + "uid": "dd03d369-7873-4514-b6fc-aadbb35ce0ab" + }, + "source_schema_sha256": "7a2d605a677a462ca619dba594ce7ebcf500358345560ad084c1b67a25c722df", + "spark_host_image_id": "sha256:f201367640c7583add224796a629150e63d3859ddd7fe9fd47741662a6d415bb" + }, + "ledger_counts": { + "approvals": 1, + "artifacts": 5, + "attempts": 2, + "evaluator_evidence": 1, + "execution_plans": 1, + "lineage_events": 1, + "policy_decisions": 1, + "quality_results": 1, + "run_events": 4 + }, + "lineage_event_id": "76548f25-e8cc-561f-ad68-56bbc4b2d9b8", + "namespace_retention": { + "cleanup_command_recorded": true, + "expires_at": "2026-08-07T04:15:23.082316Z", + "name": "gda-metadata-spark-object-store", + "owner": "metadata-platform", + "retention_id": "m3-24-229740ac50ebb53b", + "uid": "824a5904-70cc-4a85-8503-ca83acbcde16" + }, + "object_store_prepared": { + "bucket": "gda-metadata-warehouse", + "created": true, + "head_bucket_verified": true, + "material_recorded": false, + "path_style_access": true, + "region": "us-east-1", + "service": "metadata-object-store" + }, + "oidc_verified": false, + "output_artifact_id": "1f73db93-92e7-5970-842f-2477d6de394b", + "output_content_sha256": "bdc06792e8b935176ee6df6f6f6d4be1535622d54d9b994a778cabfe5a574618", + "output_resource_version_id": "a6000000-0000-4000-8000-000000000002", + "platform_run_state_version": 3, + "platform_run_status": "succeeded", + "platform_run_succeeded": true, + "production_ingestion_verified": false, + "production_object_store_verified": false, + "production_ready": false, + "production_scheduler_verified": false, + "production_tenant_attestation_verified": false, + "protected_workload_identity_verified": false, + "provider_observation": { + "external_namespace": "180265934794176", + "external_run_id": "1", + "observation_id": "85bead83-1901-5649-b6e4-fe46d01c9ea9", + "observation_sha256": "eed782687756dcddbf7f0966f70902edd037f0fc1e2acf3bb72a860e780d0819", + "observed_state": "success", + "provider_state": "SUCCESS" + }, + "quality_evidence_artifact_id": "130ee0f2-a817-5cd7-bb00-9bb79ff6aec4", + "quality_result_id": "80653601-7b44-55ca-b535-1e0ff080b17f", + "retained_control_database_verified": true, + "retained_staging_material_verified": true, + "retention_id": "m3-24-229740ac50ebb53b", + "retention_observation": { + "control_database_ref": "docker:gda-m3-24-control-229740ac50ebb53b/postgres", + "data_file_count": 1, + "data_size_bytes": 94603, + "expires_at": "2026-08-07T04:15:23.082316Z", + "feature_count": 20, + "materialized_at": "2026-07-31T04:16:04.123191Z", + "metadata_body_sha256": "6cceb8ba61378122a989e9b6046c328c517d134ee4513af5697d92c672ab772c", + "namespace": "gda-metadata-spark-object-store", + "namespace_uid": "824a5904-70cc-4a85-8503-ca83acbcde16", + "object_inventory_sha256": "de4a0efed9fdb68f0019b843377f6c8de71664de955130d0dd38e99eccdb8034", + "observation_sha256": "66000897ae58d30c84c60ee8bae2162b9fe32387fe75c8605c114dad297978c8", + "observed_at": "2026-07-31T04:16:05.189349Z", + "output_content_sha256": "bdc06792e8b935176ee6df6f6f6d4be1535622d54d9b994a778cabfe5a574618", + "output_resource_version_id": "a6000000-0000-4000-8000-000000000002", + "owner": "team:metadata-platform", + "readable": true, + "retention_id": "m3-24-229740ac50ebb53b", + "row_set_sha256": "c26ff708f4b6be082327dff63a6a8659420dbc4cab37dea1cac7b40f147512df", + "run_id": "a9000000-0000-4000-8000-000000000009", + "schema": "gda.retained_real_feature_material_observation.v1", + "snapshot_id": 8034081021802585202, + "source_payload_retained": false, + "storage_uri": "s3://gda-metadata-warehouse/warehouse/cultural_heritage/cultural_districts", + "tenant_id": "metadata-authorization-local" + }, + "retention_observation_sha256": "66000897ae58d30c84c60ee8bae2162b9fe32387fe75c8605c114dad297978c8", + "run_id": "a9000000-0000-4000-8000-000000000009", + "runtime_port_forwards_stopped": true, + "scheduler_container_cleanup_verified": true, + "schema": "gda.retained_real_feature_terminal_success_evidence.v1", + "source_absolute_path_committed": false, + "source_dataset_committed": false, + "source_feature_payload_committed": false, + "source_ingestion_evidence_sha256": "42abd82613eaf28cb53c64280258bc75dba6cf841f9a513a4c801a9f798b9899", + "source_payload_removed_from_runtime": true, + "source_projection": { + "bounds": [ + 106.37987914500007, + 29.558008447000077, + 106.59532712300006, + 29.87727198500005 + ], + "feature_count": 20, + "geometry_types": [ + "MultiPolygon", + "Polygon" + ], + "geometry_z_count": 20, + "non_empty_geometry_count": 20, + "payload_sha256": "f60e33c9ef890eb20c87e6cde82e658aaa3b20151e528ffbb293b9b64fabbbe1", + "payload_size_bytes": 430066, + "row_set_sha256": "c26ff708f4b6be082327dff63a6a8659420dbc4cab37dea1cac7b40f147512df", + "row_sha256": [ + "0527a75a86c70503302f673535003792e135c0097fcb3a44ec3b081ab74e4a76", + "08113e2f628a656b9e3a23ada3b94e0c7568ec7585ae9f25efe7702662d1f2fd", + "176093ff06f206a65fa98bcc0cb0851cb66b91b8f223d28c1b60317298ca9190", + "2305d14edf0ba61478a7e8d0511d904e36574dd691c7e580026416ed891cc18a", + "307531c08ac817f34fd4f40b2f0e9afc96758e67a4a9af2c199a9966242ffc88", + "492fee3e36f8a4ba7919ffcd6b9c4d1977baf335670a7d81159e55ee89d38506", + "4939fa3bb87d941c8a5a3e328f45bb0077982fa7d172968ed9afc76495a0a8bd", + "52de730203b330e1fa66790c2f6670d3248c0cb2620788c045e26aba3f4263a0", + "62b781771b8aa704b34fe556ace2afa47e843611fbf82c4f487ac5323a956952", + "6e96076e7ee2cfbc37b5bd093eedcd1bc9a03a4618fb30f5a7d63581aad77ca9", + "7c5d3112549cfc3b8c95cad896179060d2c9b9c5e56edd9646026ba2c9ffc039", + "8d346f09d4b606cd7ade5b196d10a3a5f54b3dff40205f1e21bea4aae10df5da", + "947b9485ed7d084e02ef780e82fd6a4fb1c37c601fe19a4be37962cd5526979b", + "a55e54b78a5b64ee6e629586968f7b304521b28be94b55b9c7ebe8670e0b83db", + "c284f86d861e265b918fd14a3f5653133bbdcd01f7f52d1cc3cde509044106ff", + "c55ed9b57879c11ea1f606e3318147893d54a4f02e9e96d4948e92cfc0880eb7", + "c84e602b9cb743f5f40b4e7c93ee4322e62c23edecf90d4b6d14c441e7c5113f", + "ce29800e1a902c1c1435277f328b599a05a7d431543cf9994ef2528f7913e34f", + "e12c4abaa5df83bdccfc33852a2f6e2e2483aa3c0ce2e77e0f470e9e76113968", + "e271fe60f1ecf40813c21b727a88b9f95b2a07c5e8df797d9370414e1a161d23" + ], + "schema": "gda.real_feature_row_set.v1", + "source_payload_recorded": false, + "srid": 4490, + "unique_identifier_count": 20, + "valid_geometry_count": 20 + }, + "source_promotion_evidence_sha256": "f6efea5000791dec1716a8354a8e39a8425b083ca4d409f4bcb61f0e7e03580d", + "source_resource_version_id": "a6000000-0000-4000-8000-000000000001", + "status": "local_retained_real_feature_terminal_success_verified", + "tenant_id": "metadata-authorization-local", + "tls_verified": false, + "writes_to_legacy": false +} diff --git a/docs/roadmap.md b/docs/roadmap.md index ac12c50c..55355a04 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -419,7 +419,7 @@ AR-0 Architecture / Schema / Runtime Truth 当前 Metadata Fabric 证据边界：M1 只读 bridge 合同已验证；ADR-037 至 ADR-046 分别覆盖本地 foundation/recovery/metrics/network-policy 演练与 production readiness contracts；ADR-047 至 ADR-050 已依次建立 deterministic projection plan、本地双 provider replay、tenant-scoped binding ledger 与本地 OpenLineage 幂等 wire delivery；ADR-051 以临时非管理员 OpenMetadata bot 证明项目专用 grant 只有 `table/Create`、`policy/Create` 被 403 拒绝，且 JWT 轮换/吊销后旧值/当前值均返回 401；ADR-052 又在隔离 Gravitino `1.3.0` Basic IdP 中证明 bounded user 的 `USE_CATALOG`、`USE_SCHEMA`、`CREATE_TABLE` 范围、catalog-create 403、密码轮换和用户吊销；ADR-053 将生产 OIDC federation、双 provider integration/workload identity、最小权限、TLS/mTLS、持久 Gravitino catalog、tenant isolation、运营责任和新鲜 protected attestation 冻结为 fail-closed readiness contract。Gravitino `1.3.0` 镜像只发现 Basic IdP，不假设 native OIDC；当前 profile 仍有 40 个外部 blockers 且未提交真实 attestation。`local_openmetadata_minimum_privilege_verified=true` 与 `local_gravitino_minimum_privilege_verified=true` 都只描述各自临时 provider rehearsal；M3-2 ingestion 仍使用 bootstrap admin，Gravitino probe catalog 仍是 memory catalog。因此 `provider_minimum_privilege_verified`、protected workload identity、OIDC、TLS、持久 catalog、生产 ingestion/conformance、生产 lineage receiver、`production_identity_gate_passed` 与 `production_ready` 仍为 false。 -M3-22/M3-23 已把一份真实重庆 20-feature EPSG:4490 slice 从受授权 Spark/Sedona + JDBC/S3 Iceberg ingestion 推进到临时 GDA Control 的原子 `ResourceVersion + 2 Artifacts + QualityResult + LineageEvent` 晋级，并验证失败整笔回滚、精确 replay、FORCE RLS、跨租户和 direct mutation 拒绝。该结果仍限定在已删除的本地 material 与临时 PostgreSQL；Run 保持 `accepted@0`，完整 PolicyDecision/Approval Artifact、成功 observation、独立 quality evidence provenance、持久 staging material 和生产 identity/storage/tenant attestation 仍是下一门槛。 +M3-22 至 M3-24 已把一份真实重庆 20-feature EPSG:4490 slice 从受授权 Spark/Sedona + JDBC/S3 Iceberg ingestion，推进到保留 7 天的 staging material、原子 GDA Control `ResourceVersion + 2 Artifacts + QualityResult + LineageEvent` 晋级和数据库裁决的 `succeeded@3`。M3-24 已持久化完整 execution-plan/PolicyDecision/Approval Artifacts，回读真实 DolphinScheduler `SUCCESS`，由独立 evaluator 重开 Parquet 并重算九项空间质量与 row fingerprint，且精确 terminal replay 不新增事实；源 payload ConfigMap 与临时 scheduler 已清理，namespace/PVC/MinIO-Iceberg material 和专用 GDA Control PostgreSQL 受同一 retention ID 约束保留。该结果仍是单开发主机上的 retained local staging rehearsal；生产 identity/storage/tenant attestation、常驻 scheduler/executor、独立故障域、backup/PITR、restart/recovery、staging scale 和完整 Spark/Flink conformance 是下一门槛。边界见 [ADR-070](architecture-decisions/adr-070-retained-real-feature-terminal-success.md)。 ### AR-2 — Source, Ingestion and Geospatial Lakehouse Vertical Slice（P0） @@ -686,7 +686,7 @@ Golden checks 至少覆盖： 1. 导出所有目标环境 schema/config fingerprint，修复重复 migration ID、checksum 和 fail-open runner。 2. 完成部署、存储、bucket、registry、scheduler/job、API/GIS endpoint、图层/样式/缓存、provider/Gateway、数据资产、消费者和权限事实盘点；部署 OpenMetadata/Gravitino/DolphinScheduler/Temporal sandbox，冻结 owner、version、OIDC、backup/restore 和升级责任。 3. 冻结 ResourceURN、ResourceVersion、PlatformDefinition/PlatformRun/FrameworkAttemptObservation/Artifact/LineageEvent、SubjectContext 与 storage/table/compute provider 最小合同。 -4. 分阶段实现 `gda-metadata-fabric-bridge`：M1 只读 mapping/reconciliation、M2a 本地 foundation/重启连续性、M2b-1 本地三存储恢复、M2b-2 隔离 versioned/Object-Locked repository round-trip、M2b-3 本机双集群 + Kubernetes 外 COMPLIANCE repository + 独立 writer/reader、M2c-1 provider-native metrics、M2c-2 临时 OTel Collector + JSON Exporter 的双周期本地 pipeline，以及 M2c-3 本地单 job scrape 故障检测/配置恢复/完整清理 evidence 已验证。下一步完成 source host/cluster 外的生产 bucket、KMS/TLS/workload identity、source-loss recovery 与 RPO/RTO，并冻结 metrics backend、retention、OTel/TLS、tenant、alert/SLO/owner 后验证持续采集、存储、查询、真实告警投递与 runbook 响应；再推进 OIDC、upgrade/rollback、NetworkPolicy enforcement、registry provenance 和 owner/runbook；之后才进入 M3 ingestion/OpenLineage/conformance。 +4. 分阶段实现 `gda-metadata-fabric-bridge`：M1 只读 mapping/reconciliation、M2a 本地 foundation/重启连续性、M2b 本地与跨集群恢复、M2c metrics/OTel 故障演练、M2d production readiness contracts，以及 M3-1 至 M3-24 projection、provider identity/interoperability、Active Metadata、真实 feature ingestion、原子 ledger promotion 和 retained terminal success 已验证。下一步以 production identity/storage/tenant attestation 为先决条件，部署常驻 scheduler/executor 与持久 catalog/control/storage，验证 restart/recovery、backup/PITR、独立故障域、staging scale、完整 Spark/Flink conformance 和真实告警/runbook；本地 retained evidence 不计入生产退出门。 5. 实现 `gda-orchestration-gateway`、DolphinScheduler process/task/schedule/complement/worker-group、Spark/Flink provider task adapter 和故障注入；不再开发新的 lease/queue/scheduler。 6. 冻结首条地类图斑数据、标准版本、敏感级别、owner、SLO 和 golden result。 7. 冻结 Default Lakehouse、Cloud Managed、Lightweight Integrated profiles；以统一 Run 完成默认 MinIO/Iceberg/Spark/Flink、轻量 PostGIS/DuckDB 和 Azure 代表 adapter 的 conformance smoke。 diff --git a/docs/system-of-record-matrix-2026-07-24.md b/docs/system-of-record-matrix-2026-07-24.md index b0fa1128..aa89ce54 100644 --- a/docs/system-of-record-matrix-2026-07-24.md +++ b/docs/system-of-record-matrix-2026-07-24.md @@ -2,9 +2,9 @@ 日期：2026-07-31 -阶段：AR-0 `in_progress`；AR-1 gateway、成功终局 evidence gate、DolphinScheduler adapter sandbox POC、Metadata Fabric M1/M2、M2c-4/M2d-2 production readiness contracts、M3-1 至 M3-23 local real-feature ledger promotion 已验证，可保留 output material、完整授权/成功/质量 provenance、生产观测、生产 policy/tenant isolation、生产 identity/object-store attestation、生产 consumer/scheduler/executor 和生产切换仍 `in_progress` +阶段：AR-0 `in_progress`；AR-1 gateway、成功终局 evidence gate、DolphinScheduler adapter sandbox POC、Metadata Fabric M1/M2、M2c-4/M2d-2 production readiness contracts、M3-1 至 M3-24 retained real-feature terminal success 已验证；生产 policy/tenant isolation、生产 identity/object-store attestation、常驻 production consumer/scheduler/executor、restart/recovery、scale/conformance 和生产切换仍 `in_progress` -适用分支：`feat/ar1-metadata-fabric-real-feature-ledger-promotion` +适用分支：`feat/ar1-metadata-fabric-retained-real-feature-terminal-success` ## 判定规则 @@ -20,19 +20,19 @@ | SQL schema 历史 | PostgreSQL `schema_migrations`，以完整 migration ID + checksum 为权威 | migration CLI 的 JSON 报告 | 保持现有 ledger；任何 drift fail closed | Data Platform | AR-0，已验证 | | 部署配置策略 | Compose/K8s/进程环境；`platform_truth.CONFIG_SPECS` 定义关键类型与策略；DolphinScheduler worker 与 Active Metadata consumer 均有默认零副本、外部 ConfigMap/Secret 驱动的 Kustomize 模板和静态 validator，前者另有 staging activation preflight | `.env` 仅补默认；脱敏 snapshot、Secret key attestation、未扩容 Deployment 和 `ready_for_activation` 都是观测/模板 | 版本化 DeploymentProfile + secret reference；部署环境始终优先；模板或 preflight 通过都不等于环境已启用 | Platform/SRE/Security | AR-0，部分实现；worker 模板/preflight 本地已验证 | | 环境发布与晋级 | 本地 candidate/registry/provenance/release/live 合同已绑定 publisher、verifier、OCI 和 manifest identity；canonical `main@0182406`、archive refs、三组 active ruleset 与 `staging-provenance` protected environment 已建立，但尚无成功 publisher/verifier 或 deployment | 旧 mainline、feature branch、CI artifact、JSON、离线 report 和合成 `verified_for_staging_apply` 都不能单独成为发布权威；publisher SHA、verifier SHA 与 branch lineage 必须分别验证 | 由受保护 environment 的 DeploymentRevision 绑定 OCI、provenance artifact、release manifest 与全部 live verdict | Platform/SRE/Security/Repository Owner | AR-1 mainline 治理已恢复 -> 首次 GHCR publish/verify -> 真实 staging | -| 后台运行时清单 | `platform_truth.RUNTIME_INVENTORY` 是代码层登记；`gda_control` 已有受控 PlatformRun 写入口；DolphinScheduler managed worker 已登记但尚无生产调用方；Active Metadata consumer 登记为 `activation_request_staging_only`，其 deployment 默认为 0 replicas；Metadata Fabric recovery/metrics/policy/catalog/interoperability/failure/uncertain-commit/outbox/consumer/authorization/scheduler-delivery/projection-execution/binding-reconciliation/real-feature-ingestion/ledger-promotion rehearsal 均为 `local_verification_only` | AST primitive report、worker status JSON、FrameworkAttemptObservation、DolphinScheduler instance state、本地 recovery/metrics/network-policy/catalog/interoperability/failure/outbox/consumer/authorization/delivery/projection/binding/ingestion/promotion evidence | PlatformRun ledger 唯一登记最终状态；M3-23 临时 promotion commit 仍把 Run 留在 `accepted@0`，不是平台成功终局权威；本地演练进程、已删除 material 与 evidence 不得变成生产控制器、监控后端、catalog authority、output authority 或 tenant-isolation 权威 | Platform Architecture | AR-1 adapter/worker、M3-15 至 M3-23 本地控制链已验证；常驻受保护 executor、retained material 与完整 terminal evidence -> staging 待接入 | +| 后台运行时清单 | `platform_truth.RUNTIME_INVENTORY` 是代码层登记；`gda_control` 已有受控 PlatformRun 写入口；DolphinScheduler managed worker 已登记但尚无生产调用方；Active Metadata consumer 登记为 `activation_request_staging_only`，其 deployment 默认为 0 replicas；M3-24 retained terminal-success rehearsal 与此前 Metadata Fabric rehearsals 均登记为 `local_verification_only` | AST primitive report、worker status JSON、FrameworkAttemptObservation、DolphinScheduler instance state、本地 recovery/metrics/network-policy/catalog/interoperability/failure/outbox/consumer/authorization/delivery/projection/binding/ingestion/promotion/terminal evidence | PlatformRun ledger 唯一登记最终状态；M3-24 的 Run 已由数据库 evidence gate 裁决为 `succeeded@3`，但临时 scheduler/executor、单主机 retained namespace/PVC/MinIO-Iceberg/control DB 和本地 evidence 不得变成生产控制器、catalog/storage/output 或 tenant-isolation 权威 | Platform Architecture | AR-1 adapter/worker、M3-15 至 M3-24 本地控制链和 retained terminal success 已验证；常驻受保护 scheduler/executor、生产 storage/control 与 restart/recovery 待接入 | | 原始文件/对象 | 当前 local uploads、S3/MinIO/OBS 均可能被直接写入，权威边界未统一 | 临时上传、下载缓存、预览文件 | Landing object 以 immutable URI + checksum + retention 为权威；本地 scratch 可删除 | Data Platform | AR-2 | | 湖仓表与 snapshot | Iceberg/STAC/S3A 有局部实现，尚无通用发布权威 | STAC item、GeoParquet export | Iceberg catalog snapshot 是分析表版本权威；对象是物理内容，STAC 是发现投影 | Data Platform | AR-2 | | 在线空间数据 | PostGIS 业务表是当前编辑/查询事实，部分临时表混入 | Martin MVT、API JSON、导出文件 | 已批准 DataProductVersion 物化到 PostGIS；不能由瓦片或临时表反向定义产品版本 | GIS/Data Platform | AR-2 -> AR-4 | -| 数据资产身份与版本 | `gda_control.resource/resource_version` 已实现 identity、hash、predecessor、tenant FK 和幂等 gateway 写入；M3-14 将新 ResourceVersion 与变化事件同事务创建；M3-23 又将 output ResourceVersion 与 2 Artifacts、QualityResult、LineageEvent 作为一个 exact-replay bundle 原子追加并拒绝半状态；`agent_data_assets`、`agent_asset_versions` 仍是兼容写路径 | UI catalog、search index、STAC、Active Metadata delivery state | GDA ledger 管身份与版本绑定；authority Resource 必须预先存在，旧行不得伪装成实时事件或部分 promotion；OpenMetadata 管治理目录，Gravitino 管技术对象映射 | Metadata Platform | AR-1 gateway/M3-14/M3-23 本地事务已验证 -> retained staging/生产写入口切换待验收 | +| 数据资产身份与版本 | `gda_control.resource/resource_version` 已实现 identity、hash、predecessor、tenant FK 和幂等 gateway 写入；M3-14 将新 ResourceVersion 与变化事件同事务创建；M3-23 建立 exact-replay output bundle promoter，M3-24 又对仍可读的 retained material 原子追加同类 bundle 并完成终局 | UI catalog、search index、STAC、Active Metadata delivery state | GDA ledger 管身份与版本绑定；authority Resource 必须预先存在，旧行不得伪装成实时事件或部分 promotion；OpenMetadata 管治理目录，Gravitino 管技术对象映射；本地 retained version 不自动成为生产版本 | Metadata Platform | AR-1 gateway/M3-14/M3-24 retained GDA Control 已验证 -> 生产写入口与 storage authority 切换待验收 | | 技术元数据 | M1 已冻结 Gravitino table ref/reconciliation；M2 已验证本地 foundation/recovery/metrics/policy 和 production readiness contracts；M3-1 固定 technical projection intent，M3-2 已在 Gravitino memory catalog 创建/read-back，M3-3 将验证后的 ref 追加到 tenant-scoped 本地 binding ledger；M3-6 又在隔离 Gravitino Basic IdP 中验证 bounded table-create、catalog-create 拒绝、密码轮换/用户吊销和完整清理；M3-7 已冻结 production identity profile/attestation gate；M3-8 已验证同一 Basic role 与 Iceberg JDBC table 在 PostgreSQL/Gravitino Pod restart 后保持；M3-9 已验证 Spark 经标准 Iceberg REST 对同一 JDBC catalog 做 read/write/schema evolution/snapshot/time travel；M3-10 已移除共享 warehouse PVC，并由跨节点 MinIO 对象检查与 Gravitino API 回读验证 Spark 结果；M3-11 已冻结 provider-neutral production object-store profile/attestation gate；M3-12 已验证 pre-forward 503 下失败提交零可见漂移、一次显式重试和无孤儿 data file | harvester 结果、合成 response、本地 sandbox/recovery/metrics/policy/ingestion/identity/JDBC restart/Spark/object-store/commit-failure observation、projection plan、provider evidence、binding ledger 与 readiness report | 源系统技术对象是原始证据；Gravitino 映射并联邦，不能覆盖业务 ResourceVersion；GDA binding ledger 只记录已验证关系，本地 memory/JDBC/file-PVC/MinIO catalog evidence 不得冒充生产持久技术权威；Basic IdP、无认证 REST/HTTP、本地主机对象存储、pending profile 和合成 attestation 都不是生产身份或生产 storage，M3-11 合同不构成 provider selection/deployment，M3-12 本地证据不构成网络 exactly-once 或生产 reconcile | Metadata Platform | AR-1 M1/M2 + M3-6 identity + M3-7 gate + M3-8 local persistence + M3-9/M3-10 interoperability + M3-11 object-store gate + M3-12 commit-failure recovery 已验证 -> 受保护身份/生产对象存储 attestation/uncertain outcome reconcile/完整 Spark-Flink conformance 待执行 | | 治理目录 | M1 已冻结 OpenMetadata table ref/reconciliation；M2 已验证本地 foundation/recovery/metrics/policy 和 production readiness contracts；M3-2 已用 bootstrap admin 创建目标并回读真实 UUID；M3-3 将该 UUID 经 evidence gate 追加到本地 GDA binding ledger；M3-4 将精确 OpenLineage candidate 经 outbox 投递到本地 HTTP receiver；M3-5 已验证临时非管理员 bot 的 scoped `table/Create` grant、policy-create 拒绝及 JWT 轮换/吊销；M3-7 将其 allow/deny 范围纳入双 provider production identity gate | 搜索/页面视图、合成 response、本地 sandbox/recovery/metrics/policy/ingestion/identity observation、projection/provider evidence、binding ledger、lineage outbox/receipt、OpenLineage event 与 readiness report | OpenMetadata 为 owner/glossary/classification/quality discoverability 权威；GDA ledger 保留审批/provider identity，outbox 只拥有投递状态，receiver 拥有接收状态；pending identity profile 和合成 attestation 均不反写 ResourceVersion 或建立生产权威 | Governance | AR-1 M1/M2 + M3-5 local bounded identity + M3-7 readiness contract 已验证 -> protected identity ingestion/生产持久 binding/受保护 production receiver 待执行 | -| 血缘 | `gda_control.lineage_event` 已实现 immutable version edge 和幂等 gateway ingest；M3-23 已将真实 source/output version edge 与 output Artifact 在同一 promotion 事务写入；`agent_asset_lineage` 旧记录仍是可变 asset edge | OpenMetadata lineage graph、UI DAG | 只有 source/target ResourceVersion、Run input、Definition、Artifact 与 event checksum 完整匹配才可形成权威 edge；目录图只作可重建投影 | Data Platform | AR-1 gateway/M3-23 本地 PostgreSQL 已验证 -> staging adapter 待接入 | +| 血缘 | `gda_control.lineage_event` 已实现 immutable version edge 和幂等 gateway ingest；M3-24 已将真实 source/output version edge 与 retained output Artifact 在同一 promotion 事务写入并纳入终局 gate；`agent_asset_lineage` 旧记录仍是可变 asset edge | OpenMetadata lineage graph、UI DAG | 只有 source/target ResourceVersion、Run input、Definition、Artifact 与 event checksum 完整匹配才可形成权威 edge；目录图只作可重建投影 | Data Platform | AR-1 gateway/M3-24 retained PostgreSQL 已验证 -> 生产 lineage adapter 待接入 | | Definition | `gda_control.platform_definition_version` 已绑定 definition ResourceVersion、完整逻辑 hash 和原子 gateway registration；3.4.2 adapter 可编译、创建并上线 provider DAG；binding 已以 append-only `execution_plan` Artifact 持久化并可按 tenant + artifact UUID 读取，旧 workflow/template/YAML 仍在写入 | 编辑器状态、DolphinScheduler DAG/definition | 旧 workflow 必须规范化并完整 hash 后才可形成 PlatformDefinitionVersion；provider binding 作为 ExecutionPlanArtifact/evidence，不可反写 definition | DataOps | AR-1 binding persistence 代码已验证 -> staging 调用链待验收 | -| Run 最终状态 | `gda_control.platform_run/event` 已实现受控 submit/read/CAS；通用 transition 已禁止 `succeeded`，专用数据库 finalizer 只接受精确 workload、success observation、内容匹配 output、独立 passed QualityResult/evidence 和 input-to-output lineage；M3-23 即使五类 output facts 已落账，显式 finalization 仍被拒绝并保持 `accepted@0` | Redis progress、日志、DolphinScheduler state、attempt observation | 旧 run 到 PlatformRun 永久 prohibited；provider/ledger facts 只形成 observation 与 reconciliation 输入，数据库 evidence gate 唯一裁决成功 | DataOps/AgentOps | AR-1 success authority/M3-23 negative gate 本地 PostgreSQL 已验证 -> 完整 staging terminal evidence 待验收 | +| Run 最终状态 | `gda_control.platform_run/event` 已实现受控 submit/read/CAS；通用 transition 已禁止 `succeeded`，专用数据库 finalizer 只接受精确 workload、success observation、内容匹配且可读的 retained output、独立 passed QualityResult/evidence 和 input-to-output lineage；M3-24 由真实 DolphinScheduler `SUCCESS` 推进到 `reconciling@2` 后，数据库 finalizer 裁决为 `succeeded@3`，精确 replay 不新增 | Redis progress、日志、DolphinScheduler state、attempt observation | 旧 run 到 PlatformRun 永久 prohibited；provider/ledger facts 只形成 observation 与 reconciliation 输入，数据库 evidence gate 唯一裁决成功；本地成功终局不等于生产 runtime readiness | DataOps/AgentOps | AR-1 success authority/M3-24 retained real-data terminal gate 已验证 -> 生产 scheduler/executor/storage/control 与 recovery 待验收 | | 调度与补数 | APScheduler、自进化 scheduler 和调用方定时逻辑并存；DolphinScheduler POC 只验证 manual start/list/variables/STOP | UI schedule 列表 | DolphinScheduler 管 DataOps schedule/complement；Temporal 只管需要 durable signal/compensation 的 Agent/GWM workflow | DataOps/AgentOps | AR-1 manual correlation 已验证；schedule/complement/failover 待验收 | | 事件交付 | Standards outbox 已数据库耐久；`platform_command_outbox` 支持 DolphinScheduler dispatch/reconcile；M3-14 `metadata_change_outbox` 将新 ResourceVersion 与内容绑定事件同事务写入；M3-15 managed consumer 同事务创建 inert request；M3-16 将真实 ResourceVersion、Definition/Run/plan/PolicyDecision/Approval/authorizer 绑定后与一个 pending dispatch 同事务提交；M3-17 由既有 consumer 向本地真实 DolphinScheduler 提交并回读；M3-18 的单个任务触发独立授权的 provider apply/read-back 与零写 replay；M3-19 在 exact OpenMetadata 前提下修复缺失 Gravitino projection、零写 replay 后提交 immutable binding | command/metadata delivery status、消费者 claim、activation intent/request/authorization、FrameworkAttemptObservation、provider instance/correlation、provider apply/read-back/binding、worker status JSON、WebSocket 消息 | command/event 与源事实同事务入 outbox，幂等 consumer 交付；Active Metadata consumer 不能授权或执行；dispatch 与 provider apply 分别授权；scheduler/provider `SUCCESS` 和 binding persistence 仍须经平台终局 evidence gate | Platform/Integrations/Metadata Platform | AR-1 command worker、M3-14 至 M3-19 本地已验证 -> protected authorizer/worker/executor identity、生产 scheduler/provider 和 production scale-up 待执行 | -| 质量结果 | `gda_control.quality_result` 已提供 tenant RLS、append-only gateway 写入，绑定 Run、output ResourceVersion、rule version、verdict、metrics、evidence Artifact 和独立 evaluator；M3-23 已原子写入真实 20-feature 六项 passed metrics，但证据 Artifact creator 仍不是独立 evaluator；standards、QC、MMFE 专项结果仍未迁移 | dashboard、OpenMetadata quality summary | GDA ledger 保存产品终局所需的不可变 verdict/evidence；只有独立 evaluator 生成并绑定的 evidence 才满足 success gate；OpenMetadata 与 UI 只作可重建投影 | Governance/DataOps | AR-1/M3-23 ledger commit 已验证 -> 独立 quality evidence provenance/staging 待接入 | +| 质量结果 | `gda_control.quality_result` 已提供 tenant RLS、append-only gateway 写入，绑定 Run、output ResourceVersion、rule version、verdict、metrics、evidence Artifact 和独立 evaluator；M3-24 的独立 evaluator 已重开唯一 retained Parquet，重算 row fingerprint 与九项各为 20 的空间质量，并创建自己的 evidence Artifact；standards、QC、MMFE 专项结果仍未迁移 | dashboard、OpenMetadata quality summary | GDA ledger 保存产品终局所需的不可变 verdict/evidence；evidence creator 必须等于 evaluator 且不同于 output creator；OpenMetadata 与 UI 只作可重建投影 | Governance/DataOps | AR-1/M3-24 retained quality provenance 已验证 -> 生产 evaluator identity、规则集与 scale 待验收 | | 标准与语义定义 | `std_*`、semantic registry 和 YAML 共同存在，生命周期未统一 | prompt/context、搜索索引 | 版本化 Standard/SemanticDefinition 经审批后为权威；Agent context 只消费批准版本 | Governance | AR-1 -> AR-3 | | 身份与权限 | Chainlit user 可显式绑定 tenant；versioned API 从认证 principal 派生 SubjectContext；`gda_control_gateway` 是 non-login/non-bypass 最小权限角色；Run 可引用强类型 PolicyDecision/Approval Artifact；M3-5/M3-6 分别验证本地 provider scoped grant、越权拒绝和 credential rotation/revocation；M3-7 已冻结生产 OIDC/workload/tenant binding、TLS、持久 catalog 与 attestation contract，但 40 个外部输入仍 blocked；M3-8 证明同一 Gravitino Basic role 在本地 JDBC restart 后连续 | session/cache、前端菜单权限、本地 provider identity/JDBC restart evidence、pending profile 与合成 readiness report | IdP/workload identity 提供真实 service identity；PolicyDecision/Approval 继续绑定不可变资源与 execution plan；只有 fresh protected attestation 可派生双 provider production identity claims，profile、Basic/JWT evidence、restart continuity 或人工批准均不可替代 | Security | AR-1 local identities/persistence + production readiness contract 已验证 -> protected 双 provider IAM/attestation 待执行 | | GIS 服务定义与 active revision | Martin、REST/MVT/STAC endpoints 和配置直接暴露 | Ingress、tile cache、客户端图层 | GIS Service Control Plane 管 Service/Layer/Style/TMS/DeploymentRevision；provider/Gateway 仅执行 | GIS Platform | AR-4 | @@ -67,6 +67,7 @@ 22. M3-21 不修改 M3-20/M3-19 历史，而是以 M3-20 candidate 为 predecessor，将同一重庆 ResourceVersion 投影到 JDBC catalog + 跨节点 MinIO warehouse。稳定 binding 包含双 Service、三个 StatefulSet、PostgreSQL/MinIO PVC、镜像、节点和 S3 配置，Gravitino 无 warehouse PVC。首次 apply 为 1 个 table create，即时及有序重启后首个 replay 均为 `no_op/0`；直接 S3 metadata key/ETag/body SHA/表 schema 不变。该表没有 source feature rows，candidate 未落账，所有临时资源已清理。这不证明生产对象存储、durable catalog、protected identity、TLS/OIDC、生产 ingestion 或 readiness。 23. M3-22 以 M3-21 candidate 为 predecessor，将同一重庆 bundle 的 20 个真实 EPSG:4490 feature rows 规范化为八列、由精确 PolicyDecision/Approval 授权 Spark/Sedona 写入 JDBC/S3 Iceberg。六项质量计数均为 20，首次执行 `appended/1` 且只有 1 个 snapshot/Parquet，即时 replay 为 `no_op/0`；S3 直读为 1 data + 2 metadata + 2 manifest。输出 ResourceVersion、Artifact、passed QualityResult 与 LineageEvent 只是 path-free candidates，未写 GDA Control，Run 未成功终局；namespace/PV/port-forward 已清理。这不证明生产对象存储、protected identity、完整 engine conformance、生产 ingestion 或 readiness。 24. M3-23 只证明 M3-22 path-free candidates 可在临时 PostgreSQL GDA Control 中按 `ResourceVersion -> output Artifact -> quality evidence Artifact -> QualityResult -> LineageEvent` 单事务追加。缺 authority、半状态、跨租户和 direct mutation 均拒绝，故障注入整笔回滚，精确 replay 不新增；Run 保持 `accepted@0` 且 success finalization 被拒绝。M3-22 material、M3-23 数据库与完整 authorization Artifacts 均未保留或伪造补写；这不证明 persistent authority、terminal success、staging/生产 ingestion 或 readiness。 +25. M3-24 以同一真实重庆 20-feature slice 新跑一次受控链：完整 execution-plan/PolicyDecision/Approval 先落 GDA Control，DolphinScheduler Shell task 回调短生命周期 executor 执行 Spark/Sedona JDBC/S3 Iceberg ingestion，真实 `SUCCESS` 只作为 attempt observation；独立 evaluator 重开 retained Parquet 并创建质量 evidence，M3-23 promoter 原子追加 output bundle，既有数据库 finalizer 唯一裁决 `succeeded@3`，精确 terminal replay 不新增。含 BSM/WKB 的 ConfigMap 和临时 scheduler 已删除；namespace/PVC/MinIO-Iceberg material 与专用 GDA Control PostgreSQL 受 retention ID 约束保留 7 天。该单开发主机 rehearsal 不证明生产 identity/storage/tenant attestation、常驻 scheduler/executor、restart/recovery、scale/conformance 或 production readiness。 ## 已建立的 AR-0/AR-1 entry 证据 @@ -79,7 +80,7 @@ - PolicyDecision/Approval 已形成强类型、内容寻址的 append-only Artifact，PlatformRun 保存不可变 UUID 引用；gateway 在提交期校验精确资源 scope，adapter 在 dispatch 前强制 workload/evaluator identity、action、effect、有效期和审批关系，失败时不会触达 provider 或改变 Run。 - migration 095 已建立 tenant/workload-scoped dispatch/reconcile outbox；真实 PostgreSQL 测试已覆盖最小权限、Run+dispatch/callback+reconcile 原子写、完成后幂等 replay、错误 workload 空领取、lease 接管、stale worker 拒绝和 fail/retry/complete，但尚无 staging 常驻 consumer 或真实 provider callback 运行证据。 - managed DolphinScheduler command worker 已提供严格 env/config、0600 token file、tenant/workload-scoped polling、SIGINT/SIGTERM drain、interruptible wait、脱敏原子 status 和 fail-closed health CLI；默认零副本 Kustomize 模板由 Pod UID 生成 worker ID，只向 PostgreSQL NetworkPolicy 增加该 selector，主容器无原始 provider Secret、Kubernetes API token 或 RBAC；activation preflight 已固定单副本、immutable digest、ConfigMap fingerprint 和脱敏 Secret key attestation，但模板尚未在 staging/production 扩容运行。 -- migration 096 已建立 append-only QualityResult 和专用成功 finalizer；真实 PostgreSQL 16 测试已证明 gateway 不能执行私有 transition 或用通用 transition 写 `succeeded`，错误 output hash、failed quality、缺失 lineage、篡改 evidence fingerprint 均拒绝，有效证据成功且 replay 幂等。该证据仍是合成数据和本地数据库，不是 staging/生产运行证明。 +- migration 096 已建立 append-only QualityResult 和专用成功 finalizer；真实 PostgreSQL 16 测试已证明 gateway 不能执行私有 transition 或用通用 transition 写 `succeeded`，错误 output hash、failed quality、缺失 lineage、篡改 evidence fingerprint 均拒绝，有效证据成功且 replay 幂等。M3-24 又用 retained 真实 GIS output、真实 DolphinScheduler `SUCCESS` 和独立 evaluator provenance 通过同一 gate；两类结果仍是本地验证，不是生产运行证明。 - staging candidate evidence 已在本地绑定 Git SHA、本地 image ID、97/97 schema fingerprint、严格脱敏配置、runtime inventory 和 JUnit 汇总；管理员/普通角色 ledger 一致，candidate 仍固定 `staging_deployed=false`、`production_promotion_allowed=false`。GitHub Runner 和真实 staging 尚未运行该链。 - GHCR publication contract 已固定单次 application image build、OCI revision/source label、远端 raw manifest `sha256`、candidate-to-subject binding 和 GitHub OIDC provenance；独立 verifier 已区分 publisher/verifier revision，固定证书 repository/workflow/ref/digest/issuer/runner 策略并对验证 evidence 再 attested；release gate 已验证该 evidence artifact 身份并从中唯一派生 manifest image。canonical mainline、protected environment、reviewer 和 Actions 权限已配置；一次意外 publisher run 在依赖安装阶段取消，尚无真实 published/verified subject、provenance artifact 或 verified release。 - live staging collector 已对 Docker Desktop 集群完成只读实采：candidate、collection freshness、97/97 应用角色 schema、runtime baseline 和 health/readiness 通过；App/Outbox 已改为直接读取 migration ledger 并禁用 token automount，重采确认 token 隔离通过。tagged 本地镜像、缺 source/candidate/platform 注解、非 strict staging profile 及缺真实 golden-slice 仍正确阻断；合成完整 evidence 可验证 live 绑定，但因缺受保护 provenance/attestation 仍固定禁止 production promotion。 @@ -113,11 +114,12 @@ - Metadata Fabric M3-21 已把 M3-20 candidate 作为 predecessor，在 M3-10 JDBC + 跨节点 MinIO runtime 中创建同一重庆 ResourceVersion 的独立 target。受限 `gda-object-store-cultural-district-projector` 首次只创建一次表，catalog create 前后均为 403；即时和 PostgreSQL -> Gravitino restart 后首个 replay 均为 `no_op/0`。Gravitino 无 warehouse PVC；MinIO 位于 `desktop-control-plane`，provider 位于 `desktop-worker`。直接 S3 检查在 `warehouse/cultural_heritage/cultural_districts/` 下只找到 1 个 metadata JSON，无 data/manifest，key/ETag/body SHA/表 schema 重启前后相同。predecessor/logical/runtime/promotion SHA 分别为 `bb6672cb7f98fa53305e17bbca2cb5b3756d4a335a94d79114fb4184273871d1`、`614ce5e4c45dba1437dc888cbd79b2d58954184113a62c20170ab84b5570d9e1`、`dd63917b6354a2e92853763ddc3e3a981cb40717f84c0f819b1a4e6844ae100b`、`63812c311b3f239bc6a944748c4ff384250eb9c9ed9009d3384fc699f1d3eaa9`；contract/evidence SHA 为 `b1a2db34a70eaa7dd55da1d6c85da9f420c755c71868aafe7972e3794034a6cc` / `d73754c53cf16d888aa345baa5d079cc7fd98d8b84db747f52188c1a69bf1628`。candidate 未落 ledger、feature rows 未 ingest、临时资源已清理；生产对象存储与 `production_ready` 仍为 `false`。 - Metadata Fabric M3-22 已由受授权 Spark `3.5.0` + Sedona `1.9.0` 将同一重庆 bundle 的 20 个 EPSG:4490 features 写入 JDBC/S3 Iceberg。六项空间质量计数均为 20，首次 `appended/1`、1 snapshot/Parquet，即时 replay `no_op/0`；S3 直读为 1 data + 2 metadata + 2 manifest。row-set/output/contract/evidence SHA 为 `c26ff708f4b6be082327dff63a6a8659420dbc4cab37dea1cac7b40f147512df` / `bdc06792e8b935176ee6df6f6f6d4be1535622d54d9b994a778cabfe5a574618` / `af211f2d2f4830decb9ffe369cd9e7ec2c9349c2e2c8bd789347a6fdc288e1dc` / `42abd82613eaf28cb53c64280258bc75dba6cf841f9a513a4c801a9f798b9899`。output/quality/lineage 仍是 candidates，runtime/material 已清理，Run 未终局。 - Metadata Fabric M3-23 已把 M3-22 的 output ResourceVersion、output Artifact、quality evidence Artifact、独立 passed QualityResult 与 source-to-output LineageEvent 作为一个 `RunOutputLedgerPromotion`，由独立 promoter 在一个 PlatformGateway 事务中写入真实临时 PostgreSQL，同时保持此前 gateway-bound evidence 指纹不变。缺 authority 先验拒绝；QualityResult 前故障注入后候选计数全为 0；首次 `created=true`、精确 replay `created=false`，最终计数为 `1/2/1/1`。FORCE RLS、最小 grant、跨租户读/直写和八个 direct UPDATE/DELETE 拒绝通过；Run 保持 `accepted@0`，success finalization 被既有 gate 拒绝。promotion/contract/evidence SHA 为 `404b6e4e5d8194f092bd83ef99cbf2d1d727015b926cd438a79eb0210f969a22` / `bd21c81925f66acdfecca5cabd78651f31deab4165da2ccd6900c4e5796e5735` / `f6efea5000791dec1716a8354a8e39a8425b083ca4d409f4bcb61f0e7e03580d`。M3-22 material 与临时数据库已清理，完整 authorization Artifacts 未补写；`output_material_retained=false`、`platform_run_succeeded=false`、`production_ready=false`。 +- Metadata Fabric M3-24 已用新的受控执行把同一重庆 slice 推进到 retained terminal success。完整 execution-plan/PolicyDecision/Approval Artifacts 在 dispatch 前持久化；DolphinScheduler `3.4.2` task 返回真实 `SUCCESS`，provider observations 为 2；独立 evaluator 重开 94,603-byte Parquet 并使 feature/unique/non-empty/valid/Z/SRID/positive-area/bbox/row-fingerprint 九项计数均为 20。M3-23 promoter 与 migration 096 finalizer 依次提交 5 Artifacts、1 QualityResult、1 LineageEvent 和 `succeeded@3`，精确 replay 不新增。contract/evidence SHA 为 `9c8f20ca1fb9995530c4e988ced627f665857ecdf0e104bb7d07c4a4a486057a` / `d966668b5a2ea57c7a4b2a3bc9824daab9b0128d9f94e515d7be649b145de418`；retention ID 为 `m3-24-229740ac50ebb53b`，到期 `2026-08-07T04:15:23.082316Z`。source payload 与 scheduler 已清理，retained material/control DB 仍可审计；六项 production claims 与 `production_ready` 均为 `false`。 ## 下一验收证据 -- M3-21 空表 metadata promotion、M3-22 临时真实 feature ingestion 与 M3-23 临时 ledger promotion 均不计入生产对象存储、持久 authority 或 ingestion 退出门，三阶段历史 candidate/evidence 保持不变； -- M3-23 已通过 M3-22 candidates 的临时 GDA Control 原子晋级、失败回滚、精确 replay 与 security negative gates，但不计入持久/生产 authority；下一步是在可保留 staging material 上持久化完整 PolicyDecision/Approval、provider success observation 和由独立 evaluator 创建的 quality evidence，再经同一 promotion 与既有 success finalizer 完成非临时 Run 终局； +- M3-21 空表 metadata promotion、M3-22 临时真实 feature ingestion、M3-23 临时 ledger promotion 与 M3-24 retained terminal success 都不计入生产对象存储、持久生产 authority 或 ingestion 退出门，四阶段历史 candidate/evidence 保持不变； +- M3-24 已跨过 retained staging material、完整 authorization、真实 provider success observation、独立 quality evidence provenance 和 `succeeded@3` 门槛；下一步必须由受保护 production identity/storage/tenant attestation 选择生产 provider，并部署常驻 scheduler/executor、持久 catalog/control/storage，验证 restart/recovery、backup/PITR、独立故障域、staging scale 与 Spark/Flink conformance； - 完成首次 application subject publish 与 protected verifier run；当前 mainline、archive refs、ruleset、required reviewer、禁止 bypass 和 environment enable variable 已配置并复核； - 真实 provenance artifact verify、受保护 overlay 的 `verified_for_staging_apply` release report，以及 staging/production 的 schema、config/runtime snapshot、registry/live DeploymentRevision 绑定、release/live artifact attestation 和环境 compare 报告； - staging 的 migration role、应用 login membership、连接池 role/tenant 复位、双租户 API 和 success finalization 运行产物； diff --git a/scripts/metadata-fabric-retained-real-feature-terminal-success.sh b/scripts/metadata-fabric-retained-real-feature-terminal-success.sh new file mode 100755 index 00000000..fdd96001 --- /dev/null +++ b/scripts/metadata-fabric-retained-real-feature-terminal-success.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +common_git_dir="$(git -C "$repo_root" rev-parse --path-format=absolute --git-common-dir 2>/dev/null || true)" +shared_root="" +if [ -n "$common_git_dir" ]; then + shared_root="$(cd "$common_git_dir/.." && pwd)" +fi + +if [ -n "${PYTHON:-}" ]; then + : +elif [ -x "$repo_root/.venv/bin/python" ]; then + PYTHON="$repo_root/.venv/bin/python" +elif [ -n "$shared_root" ] && [ -x "$shared_root/.venv/bin/python" ]; then + PYTHON="$shared_root/.venv/bin/python" +else + PYTHON="python" +fi + +cd "$repo_root" +exec "$PYTHON" -m data_agent.metadata_fabric_retained_real_feature_terminal_success "$@"