diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e03fb8e8..594f9595 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,6 +23,7 @@ on: - feat/ar1-metadata-fabric-provider-identity - feat/ar1-metadata-fabric-gravitino-identity - feat/ar1-metadata-fabric-identity-readiness-gate + - feat/ar1-metadata-fabric-jdbc-catalog-restart env: PYTHON_VERSION: "3.13" @@ -141,6 +142,9 @@ jobs: - name: Validate metadata fabric production identity gate run: python -m data_agent.metadata_fabric_identity_gate validate + - name: Validate metadata fabric Gravitino JDBC restart evidence + run: python -m data_agent.metadata_fabric_gravitino_jdbc_restart validate + - name: Validate DolphinScheduler adapter boundary run: python -m data_agent.dolphinscheduler_adapter validate @@ -199,6 +203,7 @@ jobs: data_agent/test_metadata_fabric_provider_identity.py \ data_agent/test_metadata_fabric_gravitino_identity.py \ data_agent/test_metadata_fabric_identity_gate.py \ + data_agent/test_metadata_fabric_gravitino_jdbc_restart.py \ data_agent/test_metadata_fabric_otel_failure_rehearsal.py \ data_agent/test_metadata_fabric_otel_metrics.py \ data_agent/test_metadata_fabric_provider_metrics.py \ diff --git a/config/metadata-fabric-gravitino-jdbc-restart.local.yaml b/config/metadata-fabric-gravitino-jdbc-restart.local.yaml new file mode 100644 index 00000000..a022b16e --- /dev/null +++ b/config/metadata-fabric-gravitino-jdbc-restart.local.yaml @@ -0,0 +1,85 @@ +schema: gda.metadata_fabric_gravitino_jdbc_restart_profile.v1 +environment: local_docker_desktop + +cluster: + context: docker-desktop + source_namespace: gda-metadata-sandbox + rehearsal_namespace: gda-metadata-catalog-persistence + source_schema_configmap: metadata-gravitino-schema-1-3-0 + storage_class: standard + +runtime: + manifest: k8s/metadata-fabric-gravitino-jdbc-restart + gravitino_version: 1.3.0 + gravitino_image: gda/gravitino:1.3.0-local-arm64 + gravitino_host_image_id: sha256:d355dc7e92f9e3545d717f3eab2cbdf412115f2b82e1e544d7f6235c1eacd5a5 + gravitino_kubernetes_image_id: sha256:18e24b43be854dabdc13e96b1019eb3dc691d59cc64e411aa6a3cc49225fe2d3 + postgresql_version: 16.10-bookworm + postgresql_image: postgres:16.10-bookworm + postgresql_image_digest: sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74 + service: gravitino-persistence + service_port: 8090 + authenticator: basic + idp_extension: org.apache.gravitino.idp.web.rest.feature + access_control_enabled: true + service_account: gravitino-persistence + service_account_automount_disabled: true + transport: local_loopback_http + +dependency: + evidence_path: docs/evidence/metadata-fabric-gravitino-identity-2026-07-28.json + evidence_fingerprint: f0b0de1f80f079d43318937e0a0cc151a8546e9e307bef204738b1367f9b29fd + required_claim: local_gravitino_minimum_privilege_verified + +identity: + service_admin: gda-persistence-admin + user: gda-metadata-projection + role: gda-table-projection + material_delivery: runtime_generated_ephemeral_kubernetes_object + +catalog: + provider: lakehouse-iceberg + backend: jdbc + uri: jdbc:postgresql://gravitino-persistence-postgresql:5432/iceberg + jdbc_driver: org.postgresql.Driver + jdbc_driver_source: /opt/gravitino/libs/postgresql-42.7.0.jar + jdbc_driver_mount: /opt/gravitino/catalogs/lakehouse-iceberg/libs/postgresql-42.7.0.jar + jdbc_initialize: true + warehouse: file:///var/lib/gravitino/warehouse + postgresql_pvc: data-gravitino-persistence-postgresql-0 + warehouse_pvc: warehouse-gravitino-persistence-0 + restart_scope: postgresql_then_gravitino + +scope: + metalake: gda_persistence + catalog: lakehouse + schema: published + table: gda_persistence_probe + denied_catalog: unauthorized_catalog + role_securable_objects: + - full_name: lakehouse + type: CATALOG + privileges: + - name: USE_CATALOG + condition: ALLOW + - full_name: lakehouse.published + type: SCHEMA + privileges: + - name: CREATE_TABLE + condition: ALLOW + - name: USE_SCHEMA + condition: ALLOW + +claims: + local_gravitino_jdbc_catalog_restart_verified: false + local_authenticated_catalog_persistence_verified: false + local_postgresql_pvc_restart_verified: false + local_warehouse_pvc_restart_verified: false + persistent_catalog_identity_binding_verified: false + protected_workload_identity_verified: false + oidc_verified: false + tls_verified: false + spark_conformance_verified: false + flink_conformance_verified: false + production_ingestion_verified: false + production_ready: false diff --git a/data_agent/metadata_fabric_gravitino_jdbc_restart.py b/data_agent/metadata_fabric_gravitino_jdbc_restart.py new file mode 100644 index 00000000..388ace24 --- /dev/null +++ b/data_agent/metadata_fabric_gravitino_jdbc_restart.py @@ -0,0 +1,1504 @@ +"""Rehearse authenticated Gravitino JDBC catalog continuity across restarts. + +The local rehearsal creates a Basic-authenticated, minimum-privilege table in +an Iceberg JDBC catalog whose metadata and warehouse use isolated PVCs. It then +restarts PostgreSQL and Gravitino and requires the same bounded principal to +read the same table while an administrative mutation remains denied. The +result is local Docker Desktop evidence only, never OIDC, production durability +or Spark/Flink conformance. +""" + +from __future__ import annotations + +import argparse +import json +import secrets +import subprocess +import sys +from collections.abc import Mapping +from datetime import UTC, datetime +from pathlib import Path +from typing import Annotated, Any, Literal +from urllib.parse import quote +from uuid import UUID + +import yaml +from pydantic import BaseModel, ConfigDict, Field, SecretStr, StringConstraints + +from . import metadata_fabric_gravitino_identity as identity +from . import metadata_fabric_ingestion_replay as ingestion_replay +from . import metadata_fabric_provider_metrics as provider_metrics +from . import metadata_fabric_recovery_rehearsal as recovery + + +PROFILE_SCHEMA = "gda.metadata_fabric_gravitino_jdbc_restart_profile.v1" +CONTRACT_SCHEMA = "gda.metadata_fabric_gravitino_jdbc_restart_contract.v1" +OBSERVATION_SCHEMA = "gda.metadata_fabric_gravitino_jdbc_restart_observation.v1" +EVIDENCE_SCHEMA = "gda.metadata_fabric_gravitino_jdbc_restart_evidence.v1" +VALIDATION_SCHEMA = "gda.metadata_fabric_gravitino_jdbc_restart_validation.v1" + +CONTEXT = "docker-desktop" +SOURCE_NAMESPACE = "gda-metadata-sandbox" +REHEARSAL_NAMESPACE = "gda-metadata-catalog-persistence" +GRAVITINO_SCHEMA_SHA256 = identity.GRAVITINO_SCHEMA_SHA256 +GRAVITINO_HOST_IMAGE_ID = ( + "sha256:d355dc7e92f9e3545d717f3eab2cbdf412115f2b82e1e544d7f6235c1eacd5a5" +) +GRAVITINO_KUBERNETES_IMAGE_ID = ( + "sha256:18e24b43be854dabdc13e96b1019eb3dc691d59cc64e411aa6a3cc49225fe2d3" +) +POSTGRESQL_IMAGE_DIGEST = ( + "sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74" +) +IDENTITY_EVIDENCE_FINGERPRINT = ( + "f0b0de1f80f079d43318937e0a0cc151a8546e9e307bef204738b1367f9b29fd" +) + +REPO_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_PROFILE_PATH = ( + REPO_ROOT / "config/metadata-fabric-gravitino-jdbc-restart.local.yaml" +) +DEFAULT_EVIDENCE_PATH = ( + REPO_ROOT + / "docs/evidence/metadata-fabric-gravitino-jdbc-restart-2026-07-29.json" +) +DEFAULT_WRAPPER_PATH = REPO_ROOT / "scripts/metadata-fabric-gravitino-jdbc-restart.sh" +MANIFEST_DIR = REPO_ROOT / "k8s/metadata-fabric-gravitino-jdbc-restart" + +NonEmptyText = Annotated[ + str, + StringConstraints(strip_whitespace=True, min_length=1, max_length=1024), +] + + +class MetadataFabricGravitinoJdbcRestartError(RuntimeError): + """The local JDBC catalog restart contract failed closed.""" + + +class _FrozenModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + +class ClusterProfile(_FrozenModel): + context: Literal["docker-desktop"] + source_namespace: Literal["gda-metadata-sandbox"] + rehearsal_namespace: Literal["gda-metadata-catalog-persistence"] + source_schema_configmap: Literal["metadata-gravitino-schema-1-3-0"] + storage_class: Literal["standard"] + + +class RuntimeProfile(_FrozenModel): + manifest: Literal["k8s/metadata-fabric-gravitino-jdbc-restart"] + gravitino_version: Literal["1.3.0"] + gravitino_image: Literal["gda/gravitino:1.3.0-local-arm64"] + gravitino_host_image_id: Literal[GRAVITINO_HOST_IMAGE_ID] + gravitino_kubernetes_image_id: Literal[GRAVITINO_KUBERNETES_IMAGE_ID] + postgresql_version: Literal["16.10-bookworm"] + postgresql_image: Literal["postgres:16.10-bookworm"] + postgresql_image_digest: Literal[POSTGRESQL_IMAGE_DIGEST] + service: Literal["gravitino-persistence"] + service_port: Literal[8090] + authenticator: Literal["basic"] + idp_extension: Literal["org.apache.gravitino.idp.web.rest.feature"] + access_control_enabled: Literal[True] + service_account: Literal["gravitino-persistence"] + service_account_automount_disabled: Literal[True] + transport: Literal["local_loopback_http"] + + +class DependencyProfile(_FrozenModel): + evidence_path: Literal[ + "docs/evidence/metadata-fabric-gravitino-identity-2026-07-28.json" + ] + evidence_fingerprint: Literal[IDENTITY_EVIDENCE_FINGERPRINT] + required_claim: Literal["local_gravitino_minimum_privilege_verified"] + + +class IdentityProfile(_FrozenModel): + service_admin: Literal["gda-persistence-admin"] + user: Literal["gda-metadata-projection"] + role: Literal["gda-table-projection"] + material_delivery: Literal["runtime_generated_ephemeral_kubernetes_object"] + + +class CatalogProfile(_FrozenModel): + provider: Literal["lakehouse-iceberg"] + backend: Literal["jdbc"] + uri: Literal[ + "jdbc:postgresql://gravitino-persistence-postgresql:5432/iceberg" + ] + jdbc_driver: Literal["org.postgresql.Driver"] + jdbc_driver_source: Literal["/opt/gravitino/libs/postgresql-42.7.0.jar"] + jdbc_driver_mount: Literal[ + "/opt/gravitino/catalogs/lakehouse-iceberg/libs/postgresql-42.7.0.jar" + ] + jdbc_initialize: Literal[True] + warehouse: Literal["file:///var/lib/gravitino/warehouse"] + postgresql_pvc: Literal["data-gravitino-persistence-postgresql-0"] + warehouse_pvc: Literal["warehouse-gravitino-persistence-0"] + restart_scope: Literal["postgresql_then_gravitino"] + + +class PrivilegeProfile(_FrozenModel): + name: Literal["USE_CATALOG", "USE_SCHEMA", "CREATE_TABLE"] + condition: Literal["ALLOW"] + + +class SecurableObjectProfile(_FrozenModel): + full_name: Literal["lakehouse", "lakehouse.published"] + type: Literal["CATALOG", "SCHEMA"] + privileges: tuple[PrivilegeProfile, ...] + + +class ScopeProfile(_FrozenModel): + metalake: Literal["gda_persistence"] + catalog: Literal["lakehouse"] + schema_name: Literal["published"] = Field(alias="schema") + table: Literal["gda_persistence_probe"] + denied_catalog: Literal["unauthorized_catalog"] + role_securable_objects: tuple[SecurableObjectProfile, ...] + + +class ClaimProfile(_FrozenModel): + local_gravitino_jdbc_catalog_restart_verified: Literal[False] + local_authenticated_catalog_persistence_verified: Literal[False] + local_postgresql_pvc_restart_verified: Literal[False] + local_warehouse_pvc_restart_verified: Literal[False] + persistent_catalog_identity_binding_verified: Literal[False] + protected_workload_identity_verified: Literal[False] + oidc_verified: Literal[False] + tls_verified: Literal[False] + spark_conformance_verified: Literal[False] + flink_conformance_verified: Literal[False] + production_ingestion_verified: Literal[False] + production_ready: Literal[False] + + +class GravitinoJdbcRestartProfile(_FrozenModel): + schema_name: Literal[PROFILE_SCHEMA] = Field(alias="schema") + environment: Literal["local_docker_desktop"] + cluster: ClusterProfile + runtime: RuntimeProfile + dependency: DependencyProfile + identity: IdentityProfile + catalog: CatalogProfile + scope: ScopeProfile + claims: ClaimProfile + + +def _mapping(value: Any) -> Mapping[str, Any]: + return value if isinstance(value, Mapping) else {} + + +def _valid_uuid(value: Any) -> bool: + try: + UUID(str(value)) + except (TypeError, ValueError): + return False + return True + + +def _valid_sha256(value: Any) -> bool: + return ( + isinstance(value, str) + and len(value) == 64 + and all(character in "0123456789abcdef" for character in value) + ) + + +def _profile_securable_objects( + profile: GravitinoJdbcRestartProfile, +) -> list[dict[str, Any]]: + result: list[dict[str, Any]] = [] + for item in profile.scope.role_securable_objects: + result.append( + { + "fullName": item.full_name, + "type": item.type, + "privileges": sorted( + [entry.model_dump(mode="json") for entry in item.privileges], + key=lambda entry: entry["name"], + ), + } + ) + return sorted(result, key=lambda item: item["fullName"]) + + +def _load_dependency(profile: GravitinoJdbcRestartProfile) -> dict[str, Any]: + path = (REPO_ROOT / profile.dependency.evidence_path).resolve() + try: + path.relative_to(REPO_ROOT) + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError, json.JSONDecodeError) as exc: + raise MetadataFabricGravitinoJdbcRestartError( + "Gravitino identity dependency is unavailable" + ) from exc + if not isinstance(value, dict): + raise MetadataFabricGravitinoJdbcRestartError( + "Gravitino identity dependency is not an object" + ) + if ( + value.get("evidence_fingerprint") + != profile.dependency.evidence_fingerprint + or value.get(profile.dependency.required_claim) is not True + or value.get("production_identity_verified") is not False + or identity.verify_evidence_integrity(value) + ): + raise MetadataFabricGravitinoJdbcRestartError( + "Gravitino identity dependency does not match" + ) + return value + + +def load_profile( + path: Path = DEFAULT_PROFILE_PATH, +) -> GravitinoJdbcRestartProfile: + try: + value = yaml.safe_load(path.resolve().read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError("JDBC restart profile must be an object") + ingestion_replay._reject_sensitive_fields(value) + profile = GravitinoJdbcRestartProfile.model_validate(value) + except (OSError, TypeError, ValueError, yaml.YAMLError) as exc: + raise MetadataFabricGravitinoJdbcRestartError( + "Gravitino JDBC restart profile is invalid" + ) from exc + if _profile_securable_objects(profile) != identity._expected_securable_objects(): + raise MetadataFabricGravitinoJdbcRestartError( + "Gravitino JDBC restart role exceeds the bounded table-create scope" + ) + _load_dependency(profile) + return profile + + +def _manifest_documents() -> list[dict[str, Any]]: + documents: list[dict[str, Any]] = [] + for path in sorted(MANIFEST_DIR.glob("*.yaml")): + if path.name == "kustomization.yaml": + continue + for value in yaml.safe_load_all(path.read_text(encoding="utf-8")): + if isinstance(value, dict): + documents.append(value) + return documents + + +def _validate_manifest() -> list[str]: + errors: list[str] = [] + try: + documents = _manifest_documents() + except (OSError, yaml.YAMLError) as exc: + return [f"Gravitino JDBC restart manifest is invalid: {type(exc).__name__}"] + if any(document.get("kind") == "Secret" for document in documents): + errors.append("Gravitino JDBC restart manifest may not commit Secret values") + kinds = {str(document.get("kind")) for document in documents} + required = { + "Namespace", + "ResourceQuota", + "ServiceAccount", + "ConfigMap", + "Service", + "StatefulSet", + } + if not required.issubset(kinds): + errors.append("Gravitino JDBC restart manifest is incomplete") + rendered = json.dumps(documents, ensure_ascii=True, sort_keys=True) + markers = ( + "gravitino.authenticators = basic", + "gravitino.authorization.enable = true", + "CREATE DATABASE iceberg OWNER gravitino", + "stage-postgresql-jdbc-driver", + "postgresql-42.7.0.jar", + "volumeClaimTemplates", + "storageClassName", + "warehouse", + "automountServiceAccountToken", + "ClusterIP", + ) + for marker in markers: + if marker not in rendered: + errors.append(f"Gravitino JDBC restart manifest is missing marker: {marker}") + if "gravitino.authenticators = simple" in rendered: + errors.append("Gravitino JDBC restart may not enable simple authentication") + gravitino = next( + ( + document + for document in documents + if document.get("kind") == "StatefulSet" + and _mapping(document.get("metadata")).get("name") + == "gravitino-persistence" + ), + {}, + ) + pod_spec = _mapping( + _mapping(_mapping(gravitino.get("spec")).get("template")).get("spec") + ) + init_containers = pod_spec.get("initContainers") + init_items = init_containers if isinstance(init_containers, list) else [] + driver_init = next( + ( + _mapping(item) + for item in init_items + if _mapping(item).get("name") == "stage-postgresql-jdbc-driver" + ), + {}, + ) + resources = _mapping(driver_init.get("resources")) + if not {"cpu", "memory"}.issubset(_mapping(resources.get("requests"))) or not { + "cpu", + "memory", + }.issubset(_mapping(resources.get("limits"))): + errors.append("Gravitino JDBC driver initContainer resources are incomplete") + return errors + + +def build_contract_report( + profile_path: Path = DEFAULT_PROFILE_PATH, + wrapper_path: Path = DEFAULT_WRAPPER_PATH, +) -> dict[str, Any]: + errors: list[str] = [] + profile: GravitinoJdbcRestartProfile | None = None + try: + profile = load_profile(profile_path) + except MetadataFabricGravitinoJdbcRestartError as exc: + errors.append(str(exc)) + errors.extend(_validate_manifest()) + try: + wrapper = wrapper_path.resolve().read_text(encoding="utf-8") + for marker in ("set -euo pipefail", "metadata_fabric_gravitino_jdbc_restart"): + if marker not in wrapper: + errors.append(f"Gravitino JDBC restart wrapper is missing: {marker}") + except OSError as exc: + errors.append(f"Gravitino JDBC restart wrapper is invalid: {type(exc).__name__}") + + files: dict[str, dict[str, str]] = {} + paths = [Path(__file__).resolve(), profile_path.resolve(), wrapper_path.resolve()] + paths.extend(sorted(MANIFEST_DIR.glob("*.yaml"))) + for path in paths: + if not path.is_file(): + continue + try: + relative = path.relative_to(REPO_ROOT).as_posix() + except ValueError: + relative = path.name + files[relative] = {"path": relative, "sha256": recovery._file_sha256(path)} + + stable = { + "schema": CONTRACT_SCHEMA, + "context": CONTEXT, + "source_namespace": SOURCE_NAMESPACE, + "rehearsal_namespace": REHEARSAL_NAMESPACE, + "gravitino_version": "1.3.0", + "runtime_image_identity": { + "gravitino_host_image_id": ( + profile.runtime.gravitino_host_image_id if profile else None + ), + "gravitino_kubernetes_image_id": ( + profile.runtime.gravitino_kubernetes_image_id if profile else None + ), + "postgresql_image_digest": ( + profile.runtime.postgresql_image_digest if profile else None + ), + }, + "identity_evidence_fingerprint": IDENTITY_EVIDENCE_FINGERPRINT, + "catalog": { + "provider": profile.catalog.provider if profile else None, + "backend": profile.catalog.backend if profile else None, + "uri": profile.catalog.uri if profile else None, + "warehouse": profile.catalog.warehouse if profile else None, + "driver": profile.catalog.jdbc_driver if profile else None, + "driver_mount": profile.catalog.jdbc_driver_mount if profile else None, + "jdbc_initialize": profile.catalog.jdbc_initialize if profile else None, + }, + "restart_scope": profile.catalog.restart_scope if profile else None, + "role_securable_objects": ( + _profile_securable_objects(profile) if profile else None + ), + "local_static_contract_verified": not errors, + "local_gravitino_jdbc_catalog_restart_verified": False, + "local_authenticated_catalog_persistence_verified": False, + "persistent_catalog_identity_binding_verified": False, + "protected_workload_identity_verified": False, + "oidc_verified": False, + "tls_verified": False, + "spark_conformance_verified": False, + "flink_conformance_verified": False, + "production_ingestion_verified": False, + "production_ready": False, + "files": files, + "errors": errors, + } + return {**stable, "contract_fingerprint": recovery._canonical_sha256(stable)} + + +def _single_list_item(value: Mapping[str, Any], label: str) -> dict[str, Any]: + items = value.get("items") + if not isinstance(items, list) or len(items) != 1 or not isinstance(items[0], dict): + raise MetadataFabricGravitinoJdbcRestartError( + f"Kubernetes observation is not singular: {label}" + ) + return items[0] + + +def _container_status(pod: Mapping[str, Any], name: str) -> Mapping[str, Any]: + statuses = _mapping(pod.get("status")).get("containerStatuses") + if not isinstance(statuses, list): + return {} + return next( + (_mapping(item) for item in statuses if _mapping(item).get("name") == name), + {}, + ) + + +class IsolatedJdbcRestartRuntime: + """Own the temporary namespace, persistent volumes and restart sequence.""" + + def __init__(self, profile: GravitinoJdbcRestartProfile) -> None: + self.profile = profile + self.kubectl = identity._Kubectl(profile.cluster.context) + self.owned_namespace = False + self.schema_sha256: str | None = None + self.host_image_id: str | None = None + + def _inspect_host_image(self) -> None: + try: + completed = subprocess.run( + [ + "docker", + "image", + "inspect", + self.profile.runtime.gravitino_image, + "--format", + "{{.Id}}", + ], + check=False, + capture_output=True, + text=True, + timeout=30, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise MetadataFabricGravitinoJdbcRestartError( + "Gravitino host image identity is unavailable" + ) from exc + self.host_image_id = completed.stdout.strip() + if ( + completed.returncode != 0 + or self.host_image_id != self.profile.runtime.gravitino_host_image_id + ): + raise MetadataFabricGravitinoJdbcRestartError( + "Gravitino host image identity does not match" + ) + + def _runtime_inputs( + self, admin_material: SecretStr, database_material: SecretStr + ) -> str: + source = self.kubectl.get_json( + [ + "-n", + self.profile.cluster.source_namespace, + "get", + "configmap", + self.profile.cluster.source_schema_configmap, + ], + label="source schema lookup", + ) + assert source is not None + schema_sql = _mapping(source.get("data")).get("001-schema.sql") + if not isinstance(schema_sql, str): + raise MetadataFabricGravitinoJdbcRestartError( + "verified Gravitino PostgreSQL schema is unavailable" + ) + self.schema_sha256 = identity._sha256_text(schema_sql) + if self.schema_sha256 != GRAVITINO_SCHEMA_SHA256: + raise MetadataFabricGravitinoJdbcRestartError( + "Gravitino PostgreSQL schema checksum drift" + ) + resources = { + "apiVersion": "v1", + "kind": "List", + "items": [ + { + "apiVersion": "v1", + "kind": "Secret", + "metadata": { + "name": "gravitino-persistence-runtime", + "namespace": self.profile.cluster.rehearsal_namespace, + }, + "type": "Opaque", + "stringData": { + "admin-password": admin_material.get_secret_value(), + "database-password": database_material.get_secret_value(), + }, + }, + { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": "gravitino-persistence-schema", + "namespace": self.profile.cluster.rehearsal_namespace, + }, + "data": {"001-schema.sql": schema_sql}, + }, + ], + } + return json.dumps(resources, ensure_ascii=True, separators=(",", ":")) + + def start( + self, *, admin_material: SecretStr, database_material: SecretStr + ) -> dict[str, Any]: + self._inspect_host_image() + existing = self.kubectl.get_json( + ["get", "namespace", self.profile.cluster.rehearsal_namespace], + allow_not_found=True, + label="JDBC restart namespace preflight", + ) + if existing is not None: + raise MetadataFabricGravitinoJdbcRestartError( + "Gravitino JDBC restart namespace already exists" + ) + self.kubectl.run( + ["apply", "-f", str(MANIFEST_DIR / "namespace.yaml")], + label="JDBC restart namespace apply", + ) + self.owned_namespace = True + self.kubectl.run( + ["apply", "-f", "-"], + input_text=self._runtime_inputs(admin_material, database_material), + label="ephemeral JDBC restart inputs apply", + ) + self.kubectl.run( + ["apply", "-k", str(MANIFEST_DIR)], + label="JDBC restart runtime apply", + ) + self._wait_for_rollouts() + return self.observe() + + def _wait_for_rollouts(self) -> None: + namespace = self.profile.cluster.rehearsal_namespace + for workload in ( + "statefulset/gravitino-persistence-postgresql", + "statefulset/gravitino-persistence", + ): + self.kubectl.run( + ["-n", namespace, "rollout", "status", workload, "--timeout=10m"], + timeout=660, + label=f"{workload} rollout", + ) + + def _workload( + self, + *, + statefulset_name: str, + label_name: str, + container_name: str, + pvc_name: str, + ) -> dict[str, Any]: + namespace = self.profile.cluster.rehearsal_namespace + statefulset = self.kubectl.get_json( + ["-n", namespace, "get", "statefulset", statefulset_name], + label=f"{statefulset_name} observation", + ) + pod_list = self.kubectl.get_json( + [ + "-n", + namespace, + "get", + "pods", + "-l", + f"app.kubernetes.io/name={label_name}", + ], + label=f"{statefulset_name} pod observation", + ) + pvc = self.kubectl.get_json( + ["-n", namespace, "get", "pvc", pvc_name], + label=f"{pvc_name} observation", + ) + assert statefulset is not None and pod_list is not None and pvc is not None + pod = _single_list_item(pod_list, statefulset_name) + pod_spec = _mapping(pod.get("spec")) + status = _container_status(pod, container_name) + pvc_spec = _mapping(pvc.get("spec")) + return { + "statefulset_uid": _mapping(statefulset.get("metadata")).get("uid"), + "pod_uid": _mapping(pod.get("metadata")).get("uid"), + "pod_name": _mapping(pod.get("metadata")).get("name"), + "node_name": pod_spec.get("nodeName"), + "ready_replicas": _mapping(statefulset.get("status")).get( + "readyReplicas", 0 + ), + "service_account": pod_spec.get("serviceAccountName"), + "service_account_automount_disabled": ( + pod_spec.get("automountServiceAccountToken") is False + ), + "image": status.get("image"), + "image_id": status.get("imageID"), + "pvc": { + "name": _mapping(pvc.get("metadata")).get("name"), + "uid": _mapping(pvc.get("metadata")).get("uid"), + "storage_class": pvc_spec.get("storageClassName"), + "volume_name": pvc_spec.get("volumeName"), + "phase": _mapping(pvc.get("status")).get("phase"), + }, + } + + def observe(self) -> dict[str, Any]: + namespace_name = self.profile.cluster.rehearsal_namespace + namespace = self.kubectl.get_json( + ["get", "namespace", namespace_name], label="namespace observation" + ) + service = self.kubectl.get_json( + ["-n", namespace_name, "get", "service", self.profile.runtime.service], + label="Gravitino persistence service observation", + ) + assert namespace is not None and service is not None + driver = self.kubectl.run( + [ + "-n", + namespace_name, + "exec", + "gravitino-persistence-0", + "--", + "test", + "-r", + self.profile.catalog.jdbc_driver_mount, + ], + expected=frozenset({0, 1}), + timeout=60, + label="JDBC driver mount probe", + ) + return { + "context": self.profile.cluster.context, + "gravitino_host_image_id": self.host_image_id, + "namespace": { + "name": _mapping(namespace.get("metadata")).get("name"), + "uid": _mapping(namespace.get("metadata")).get("uid"), + }, + "service": { + "name": _mapping(service.get("metadata")).get("name"), + "uid": _mapping(service.get("metadata")).get("uid"), + "type": _mapping(service.get("spec")).get("type"), + }, + "postgresql": self._workload( + statefulset_name="gravitino-persistence-postgresql", + label_name="gravitino-persistence-postgresql", + container_name="postgresql", + pvc_name=self.profile.catalog.postgresql_pvc, + ), + "gravitino": self._workload( + statefulset_name="gravitino-persistence", + label_name="gravitino-persistence", + container_name="gravitino", + pvc_name=self.profile.catalog.warehouse_pvc, + ), + "jdbc_driver_mounted": driver.returncode == 0, + "source_schema_sha256": self.schema_sha256, + } + + def restart(self) -> dict[str, Any]: + before = self.observe() + namespace = self.profile.cluster.rehearsal_namespace + for workload in ( + "statefulset/gravitino-persistence-postgresql", + "statefulset/gravitino-persistence", + ): + self.kubectl.run( + ["-n", namespace, "rollout", "restart", workload], + label=f"{workload} restart", + ) + self.kubectl.run( + ["-n", namespace, "rollout", "status", workload, "--timeout=10m"], + timeout=660, + label=f"{workload} restart rollout", + ) + after = self.observe() + return {"before": before, "after": after} + + def cleanup(self) -> dict[str, Any]: + deleted = False + if self.owned_namespace: + try: + self.kubectl.run( + [ + "delete", + "namespace", + self.profile.cluster.rehearsal_namespace, + "--wait=true", + "--timeout=5m", + ], + timeout=330, + label="JDBC restart namespace cleanup", + ) + deleted = True + finally: + self.owned_namespace = False + absent = ( + self.kubectl.get_json( + ["get", "namespace", self.profile.cluster.rehearsal_namespace], + allow_not_found=True, + label="JDBC restart namespace cleanup verification", + ) + is None + ) + return { + "namespace_delete_completed": deleted, + "namespace_absent": absent, + "provider_objects_retained": False, + "persistent_volumes_retained": False, + } + + +def _table_projection(value: Mapping[str, Any]) -> dict[str, Any]: + columns = value.get("columns") + projected_columns: list[dict[str, Any]] = [] + if isinstance(columns, list): + for column in columns: + item = _mapping(column) + projected_columns.append( + { + "name": item.get("name"), + "type": item.get("type"), + "nullable": item.get("nullable"), + } + ) + properties = _mapping(value.get("properties")) + return { + "name": value.get("name"), + "columns": projected_columns, + "probe_property": properties.get("gda.persistence_probe"), + } + + +class PersistentCatalogRehearsal: + """Create and re-read a JDBC catalog through one bounded Basic principal.""" + + def __init__( + self, + *, + base_url: str, + admin_name: str, + admin_material: SecretStr, + ) -> None: + self.base_url = base_url + self.admin = identity._BasicApi( + base_url=base_url, + username=admin_name, + material=admin_material, + ) + self.clients: list[identity._BasicApi] = [] + + def close(self) -> None: + for client in self.clients: + client.close() + self.clients.clear() + self.admin.close() + + def _user(self, profile: GravitinoJdbcRestartProfile, material: SecretStr) -> identity._BasicApi: + client = identity._BasicApi( + base_url=self.base_url, + username=profile.identity.user, + material=material, + ) + self.clients.append(client) + return client + + @staticmethod + def _catalog_path(profile: GravitinoJdbcRestartProfile, catalog: str) -> str: + return f"metalakes/{quote(profile.scope.metalake)}/catalogs/{quote(catalog)}" + + @classmethod + def _schema_path(cls, profile: GravitinoJdbcRestartProfile) -> str: + return ( + f"{cls._catalog_path(profile, profile.scope.catalog)}/schemas/" + f"{quote(profile.scope.schema_name)}" + ) + + @classmethod + def _table_path(cls, profile: GravitinoJdbcRestartProfile) -> str: + return f"{cls._schema_path(profile)}/tables/{quote(profile.scope.table)}" + + @staticmethod + def _denied_catalog_body(profile: GravitinoJdbcRestartProfile) -> dict[str, Any]: + return { + "name": profile.scope.denied_catalog, + "type": "RELATIONAL", + "provider": "lakehouse-iceberg", + "comment": "Must remain denied after provider restart", + "properties": { + "catalog-backend": "memory", + "uri": "file:///tmp/gda-denied", + "warehouse": "file:///tmp/gda-denied", + }, + } + + def bootstrap( + self, + profile: GravitinoJdbcRestartProfile, + *, + database_material: SecretStr, + user_material: SecretStr, + ) -> dict[str, Any]: + admin_status, _version = self.admin.request( + "GET", "version", label="pre-restart admin authentication" + ) + _status, metalake_payload = self.admin.request( + "POST", + "metalakes", + json_body={ + "name": profile.scope.metalake, + "comment": "Local authenticated JDBC restart rehearsal", + "properties": {"gda.environment": "local_jdbc_restart"}, + }, + label="persistent metalake create", + ) + metalake = identity._response_entity( + metalake_payload, "metalake", "persistent metalake create" + ) + _status, catalog_payload = self.admin.request( + "POST", + f"metalakes/{quote(profile.scope.metalake)}/catalogs", + json_body={ + "name": profile.scope.catalog, + "type": "RELATIONAL", + "provider": profile.catalog.provider, + "comment": "Local JDBC-backed Iceberg persistence catalog", + "properties": { + "catalog-backend": profile.catalog.backend, + "uri": profile.catalog.uri, + "warehouse": profile.catalog.warehouse, + "jdbc-user": "gravitino", + "jdbc-password": database_material.get_secret_value(), + "gravitino.bypass.jdbc-driver": profile.catalog.jdbc_driver, + "gravitino.bypass.jdbc-initialize": "true", + }, + }, + label="JDBC catalog create", + ) + catalog = identity._response_entity(catalog_payload, "catalog", "catalog create") + _status, schema_payload = self.admin.request( + "POST", + f"{self._catalog_path(profile, profile.scope.catalog)}/schemas", + json_body={ + "name": profile.scope.schema_name, + "comment": "Persistent bounded projection schema", + "properties": {}, + }, + label="persistent schema create", + ) + schema = identity._response_entity(schema_payload, "schema", "schema create") + + self.admin.request( + "POST", + "idp/users", + json_body={ + "user": profile.identity.user, + "password": user_material.get_secret_value(), + }, + label="persistent IdP user create", + ) + self.admin.request( + "POST", + f"metalakes/{quote(profile.scope.metalake)}/users", + json_body={"name": profile.identity.user}, + label="persistent metalake user register", + ) + self.admin.request( + "POST", + f"metalakes/{quote(profile.scope.metalake)}/roles", + json_body={ + "name": profile.identity.role, + "properties": {"gda.scope": "bounded_table_projection"}, + "securableObjects": identity._expected_securable_objects(), + }, + label="persistent bounded role create", + ) + self.admin.request( + "PUT", + ( + f"metalakes/{quote(profile.scope.metalake)}/permissions/users/" + f"{quote(profile.identity.user)}/grant" + ), + json_body={"roleNames": [profile.identity.role]}, + label="persistent bounded role grant", + ) + _status, role_payload = self.admin.request( + "GET", + ( + f"metalakes/{quote(profile.scope.metalake)}/roles/" + f"{quote(profile.identity.role)}" + ), + label="pre-restart role readback", + ) + role = identity._response_entity(role_payload, "role", "role readback") + + bounded = self._user(profile, user_material) + bounded_status, _payload = bounded.request( + "GET", "version", label="pre-restart bounded authentication" + ) + create_status, table_payload = bounded.request( + "POST", + f"{self._schema_path(profile)}/tables", + json_body={ + "name": profile.scope.table, + "comment": "Authenticated JDBC catalog restart probe", + "columns": [ + { + "name": "probe_id", + "type": "string", + "nullable": False, + "comment": "Persistence probe identifier", + } + ], + "properties": {"gda.persistence_probe": "true"}, + }, + label="pre-restart table create", + ) + table = identity._response_entity(table_payload, "table", "table create") + read_status, read_payload = bounded.request( + "GET", self._table_path(profile), label="pre-restart table readback" + ) + read_table = identity._response_entity(read_payload, "table", "table readback") + denied_status, _payload = bounded.request( + "POST", + f"metalakes/{quote(profile.scope.metalake)}/catalogs", + json_body=self._denied_catalog_body(profile), + expected=frozenset({200, 403}), + label="pre-restart administrative denial", + ) + projection = _table_projection(read_table) + return { + "authentication": { + "admin_status": admin_status, + "bounded_status": bounded_status, + "material_recorded": False, + }, + "catalog": { + "metalake": metalake.get("name"), + "catalog": catalog.get("name"), + "schema": schema.get("name"), + "provider": profile.catalog.provider, + "backend": profile.catalog.backend, + "uri": profile.catalog.uri, + "warehouse": profile.catalog.warehouse, + "jdbc_initialize": profile.catalog.jdbc_initialize, + "material_recorded": False, + }, + "role": { + "name": role.get("name"), + "securable_objects": identity._normalize_securable_objects( + role.get("securableObjects") + ), + }, + "table": { + "create_status": create_status, + "read_status": read_status, + "name": table.get("name"), + "projection": projection, + "fingerprint": recovery._canonical_sha256(projection), + }, + "denied_catalog_create_status": denied_status, + } + + def verify_after_restart( + self, + profile: GravitinoJdbcRestartProfile, + *, + user_material: SecretStr, + ) -> dict[str, Any]: + admin_status, _version = self.admin.request( + "GET", "version", label="post-restart admin authentication" + ) + bounded = self._user(profile, user_material) + bounded_status, _payload = bounded.request( + "GET", "version", label="post-restart bounded authentication" + ) + read_status, read_payload = bounded.request( + "GET", self._table_path(profile), label="post-restart table readback" + ) + table = identity._response_entity( + read_payload, "table", "post-restart table readback" + ) + _status, role_payload = self.admin.request( + "GET", + ( + f"metalakes/{quote(profile.scope.metalake)}/roles/" + f"{quote(profile.identity.role)}" + ), + label="post-restart role readback", + ) + role = identity._response_entity(role_payload, "role", "post-restart role") + denied_status, _payload = bounded.request( + "POST", + f"metalakes/{quote(profile.scope.metalake)}/catalogs", + json_body=self._denied_catalog_body(profile), + expected=frozenset({200, 403}), + label="post-restart administrative denial", + ) + projection = _table_projection(table) + return { + "authentication": { + "admin_status": admin_status, + "bounded_status": bounded_status, + "material_recorded": False, + }, + "role": { + "name": role.get("name"), + "securable_objects": identity._normalize_securable_objects( + role.get("securableObjects") + ), + }, + "table": { + "read_status": read_status, + "name": table.get("name"), + "projection": projection, + "fingerprint": recovery._canonical_sha256(projection), + }, + "denied_catalog_create_status": denied_status, + } + + +def _restart_errors(restart: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + before = _mapping(restart.get("before")) + after = _mapping(restart.get("after")) + for workload_name, expected_image_id, expected_pvc, expected_account in ( + ( + "postgresql", + POSTGRESQL_IMAGE_DIGEST, + "data-gravitino-persistence-postgresql-0", + "gravitino-persistence-postgresql", + ), + ( + "gravitino", + GRAVITINO_KUBERNETES_IMAGE_ID, + "warehouse-gravitino-persistence-0", + "gravitino-persistence", + ), + ): + old = _mapping(before.get(workload_name)) + new = _mapping(after.get(workload_name)) + old_pvc = _mapping(old.get("pvc")) + new_pvc = _mapping(new.get("pvc")) + if not _valid_uuid(old.get("statefulset_uid")) or ( + old.get("statefulset_uid") != new.get("statefulset_uid") + ): + errors.append(f"{workload_name} StatefulSet identity changed") + if ( + not _valid_uuid(old.get("pod_uid")) + or not _valid_uuid(new.get("pod_uid")) + or old.get("pod_uid") == new.get("pod_uid") + ): + errors.append(f"{workload_name} pod did not restart") + if old.get("ready_replicas") != 1 or new.get("ready_replicas") != 1: + errors.append(f"{workload_name} was not ready around restart") + if not all( + str(snapshot.get("image_id") or "").endswith(expected_image_id) + for snapshot in (old, new) + ): + errors.append(f"{workload_name} runtime image ID does not match") + if old_pvc.get("name") != expected_pvc or new_pvc.get("name") != expected_pvc: + errors.append(f"{workload_name} PVC name does not match") + if not _valid_uuid(old_pvc.get("uid")) or old_pvc.get("uid") != new_pvc.get( + "uid" + ): + errors.append(f"{workload_name} PVC identity changed") + if old_pvc.get("phase") != "Bound" or new_pvc.get("phase") != "Bound": + errors.append(f"{workload_name} PVC was not bound") + if old_pvc.get("storage_class") != "standard" or new_pvc.get( + "storage_class" + ) != "standard": + errors.append(f"{workload_name} PVC storage class does not match") + if ( + old.get("service_account") != expected_account + or new.get("service_account") != expected_account + or + old.get("service_account_automount_disabled") is not True + or new.get("service_account_automount_disabled") is not True + ): + errors.append(f"{workload_name} service account token isolation failed") + for snapshot_name, snapshot in (("before", before), ("after", after)): + if ( + snapshot.get("context") != CONTEXT + or snapshot.get("gravitino_host_image_id") != GRAVITINO_HOST_IMAGE_ID + or _mapping(snapshot.get("namespace")).get("name") + != REHEARSAL_NAMESPACE + or not _valid_uuid(_mapping(snapshot.get("namespace")).get("uid")) + or _mapping(snapshot.get("service")).get("name") + != "gravitino-persistence" + or _mapping(snapshot.get("service")).get("type") != "ClusterIP" + or snapshot.get("jdbc_driver_mounted") is not True + or snapshot.get("source_schema_sha256") != GRAVITINO_SCHEMA_SHA256 + ): + errors.append(f"{snapshot_name} runtime boundary does not match") + return errors + + +def build_evidence(observation: Mapping[str, Any]) -> dict[str, Any]: + errors: list[str] = [] + try: + ingestion_replay._reject_sensitive_fields(observation) + except ValueError: + errors.append("Gravitino JDBC restart observation contains sensitive material") + if observation.get("schema") != OBSERVATION_SCHEMA: + errors.append("Gravitino JDBC restart observation schema does not match") + contract = _mapping(observation.get("contract")) + if ( + contract.get("local_static_contract_verified") is not True + or not _valid_sha256(contract.get("contract_fingerprint")) + or contract.get("identity_evidence_fingerprint") + != IDENTITY_EVIDENCE_FINGERPRINT + ): + errors.append("Gravitino JDBC restart static contract is not bound") + restart = _mapping(observation.get("restart")) + errors.extend(_restart_errors(restart)) + + pre = _mapping(observation.get("pre_restart")) + post = _mapping(observation.get("post_restart")) + pre_auth = _mapping(pre.get("authentication")) + post_auth = _mapping(post.get("authentication")) + pre_catalog = _mapping(pre.get("catalog")) + pre_role = _mapping(pre.get("role")) + post_role = _mapping(post.get("role")) + pre_table = _mapping(pre.get("table")) + post_table = _mapping(post.get("table")) + expected_projection = { + "name": "gda_persistence_probe", + "columns": [ + {"name": "probe_id", "type": "string", "nullable": False}, + ], + "probe_property": "true", + } + if ( + pre_auth.get("admin_status") != 200 + or pre_auth.get("bounded_status") != 200 + or post_auth.get("admin_status") != 200 + or post_auth.get("bounded_status") != 200 + or pre_auth.get("material_recorded") is not False + or post_auth.get("material_recorded") is not False + ): + errors.append("authenticated principals did not survive restart") + if pre_catalog != { + "metalake": "gda_persistence", + "catalog": "lakehouse", + "schema": "published", + "provider": "lakehouse-iceberg", + "backend": "jdbc", + "uri": "jdbc:postgresql://gravitino-persistence-postgresql:5432/iceberg", + "warehouse": "file:///var/lib/gravitino/warehouse", + "jdbc_initialize": True, + "material_recorded": False, + }: + errors.append("JDBC catalog configuration does not match") + expected_role = identity._expected_securable_objects() + if ( + pre_role.get("name") != "gda-table-projection" + or post_role.get("name") != "gda-table-projection" + or pre_role.get("securable_objects") != expected_role + or post_role.get("securable_objects") != expected_role + ): + errors.append("minimum-privilege role did not survive restart") + if ( + pre_table.get("create_status") != 200 + or pre_table.get("read_status") != 200 + or post_table.get("read_status") != 200 + or pre_table.get("name") != "gda_persistence_probe" + or post_table.get("name") != "gda_persistence_probe" + or pre_table.get("projection") != expected_projection + or pre_table.get("fingerprint") != post_table.get("fingerprint") + or pre_table.get("projection") != post_table.get("projection") + or pre_table.get("fingerprint") + != recovery._canonical_sha256(expected_projection) + ): + errors.append("JDBC-backed table did not survive restart") + if ( + pre.get("denied_catalog_create_status") != 403 + or post.get("denied_catalog_create_status") != 403 + ): + errors.append("administrative catalog mutation was not denied") + checks = _mapping(observation.get("runtime_checks")) + if ( + checks.get("all_port_forwards_stopped") is not True + or checks.get("namespace_delete_completed") is not True + or checks.get("namespace_absent") is not True + or checks.get("provider_objects_retained") is not False + or checks.get("persistent_volumes_retained") is not False + or checks.get("material_recorded") is not False + ): + errors.append("Gravitino JDBC restart cleanup is incomplete") + + verified = not errors + stable = { + "schema": EVIDENCE_SCHEMA, + "environment": "local_docker_desktop", + "catalog_backend": "jdbc", + "restart_scope": "postgresql_then_gravitino", + "identity_evidence_fingerprint": IDENTITY_EVIDENCE_FINGERPRINT, + "local_gravitino_jdbc_catalog_restart_verified": verified, + "local_authenticated_catalog_persistence_verified": verified, + "local_postgresql_pvc_restart_verified": verified, + "local_warehouse_pvc_restart_verified": verified, + "persistent_catalog_identity_binding_verified": False, + "protected_workload_identity_verified": False, + "oidc_verified": False, + "tls_verified": False, + "spark_conformance_verified": False, + "flink_conformance_verified": False, + "production_ingestion_verified": False, + "production_ready": False, + "observation": dict(observation), + "errors": errors, + } + return {**stable, "evidence_fingerprint": recovery._canonical_sha256(stable)} + + +def verify_evidence_integrity(evidence: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + try: + ingestion_replay._reject_sensitive_fields(evidence) + except ValueError: + errors.append("Gravitino JDBC restart evidence contains sensitive material") + stable = { + key: value for key, value in evidence.items() if key != "evidence_fingerprint" + } + if evidence.get("evidence_fingerprint") != recovery._canonical_sha256(stable): + errors.append("Gravitino JDBC restart evidence fingerprint does not match") + if evidence.get("schema") != EVIDENCE_SCHEMA or evidence.get("errors") != []: + errors.append("Gravitino JDBC restart evidence is not verified") + for claim in ( + "local_gravitino_jdbc_catalog_restart_verified", + "local_authenticated_catalog_persistence_verified", + "local_postgresql_pvc_restart_verified", + "local_warehouse_pvc_restart_verified", + ): + if evidence.get(claim) is not True: + errors.append(f"Gravitino JDBC restart evidence claim is false: {claim}") + for claim in ( + "persistent_catalog_identity_binding_verified", + "protected_workload_identity_verified", + "oidc_verified", + "tls_verified", + "spark_conformance_verified", + "flink_conformance_verified", + "production_ingestion_verified", + "production_ready", + ): + if evidence.get(claim) is not False: + errors.append(f"Gravitino JDBC restart evidence may not claim {claim}") + return errors + + +def run_live_rehearsal( + profile_path: Path = DEFAULT_PROFILE_PATH, +) -> dict[str, Any]: + profile = load_profile(profile_path) + contract = build_contract_report(profile_path) + if contract.get("local_static_contract_verified") is not True: + raise MetadataFabricGravitinoJdbcRestartError( + "Gravitino JDBC restart static contract is invalid" + ) + + admin_material = SecretStr(secrets.token_urlsafe(24)) + database_material = SecretStr(secrets.token_urlsafe(24)) + user_material = SecretStr(secrets.token_urlsafe(24)) + runtime = IsolatedJdbcRestartRuntime(profile) + before_forward: provider_metrics._PortForward | None = None + after_forward: provider_metrics._PortForward | None = None + rehearsal: PersistentCatalogRehearsal | None = None + pre_restart: dict[str, Any] | None = None + post_restart: dict[str, Any] | None = None + restart: dict[str, Any] | None = None + before_forward_stopped = False + after_forward_stopped = False + cleanup: dict[str, Any] = { + "namespace_delete_completed": False, + "namespace_absent": False, + "provider_objects_retained": True, + "persistent_volumes_retained": True, + } + try: + runtime.start( + admin_material=admin_material, + database_material=database_material, + ) + before_forward = provider_metrics._PortForward( + kubectl="kubectl", + context=profile.cluster.context, + namespace=profile.cluster.rehearsal_namespace, + service=profile.runtime.service, + target_port=profile.runtime.service_port, + ) + before_forward.start() + rehearsal = PersistentCatalogRehearsal( + base_url=f"http://127.0.0.1:{before_forward.local_port}/api", + admin_name=profile.identity.service_admin, + admin_material=admin_material, + ) + pre_restart = rehearsal.bootstrap( + profile, + database_material=database_material, + user_material=user_material, + ) + rehearsal.close() + rehearsal = None + before_forward_stopped = before_forward.stop() + before_forward = None + + restart = runtime.restart() + + after_forward = provider_metrics._PortForward( + kubectl="kubectl", + context=profile.cluster.context, + namespace=profile.cluster.rehearsal_namespace, + service=profile.runtime.service, + target_port=profile.runtime.service_port, + ) + after_forward.start() + rehearsal = PersistentCatalogRehearsal( + base_url=f"http://127.0.0.1:{after_forward.local_port}/api", + admin_name=profile.identity.service_admin, + admin_material=admin_material, + ) + post_restart = rehearsal.verify_after_restart( + profile, user_material=user_material + ) + finally: + if rehearsal is not None: + rehearsal.close() + if before_forward is not None: + before_forward_stopped = before_forward.stop() + if after_forward is not None: + after_forward_stopped = after_forward.stop() + cleanup = runtime.cleanup() + + if pre_restart is None or post_restart is None or restart is None: + raise MetadataFabricGravitinoJdbcRestartError( + "Gravitino JDBC restart rehearsal did not produce an outcome" + ) + observation = { + "schema": OBSERVATION_SCHEMA, + "observed_at": datetime.now(UTC).isoformat(), + "contract": { + "contract_fingerprint": contract["contract_fingerprint"], + "local_static_contract_verified": True, + "identity_evidence_fingerprint": IDENTITY_EVIDENCE_FINGERPRINT, + }, + "restart": restart, + "pre_restart": pre_restart, + "post_restart": post_restart, + "runtime_checks": { + **cleanup, + "all_port_forwards_stopped": ( + before_forward_stopped and after_forward_stopped + ), + "material_recorded": False, + "kubernetes_service_account_used_for_provider_login": False, + }, + } + return build_evidence(observation) + + +def build_validation_report( + *, + profile_path: Path = DEFAULT_PROFILE_PATH, + evidence_path: Path = DEFAULT_EVIDENCE_PATH, +) -> dict[str, Any]: + contract = build_contract_report(profile_path) + errors = list(contract["errors"]) + evidence: dict[str, Any] | None = None + try: + value = json.loads(evidence_path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError("Gravitino JDBC restart evidence must be an object") + evidence = value + errors.extend(verify_evidence_integrity(evidence)) + observed_contract = _mapping( + _mapping(evidence.get("observation")).get("contract") + ).get("contract_fingerprint") + if observed_contract != contract.get("contract_fingerprint"): + errors.append("Gravitino JDBC restart evidence contract fingerprint drift") + except (OSError, TypeError, ValueError, json.JSONDecodeError) as exc: + errors.append( + f"Gravitino JDBC restart evidence is invalid: {type(exc).__name__}" + ) + verified = not errors + return { + "schema": VALIDATION_SCHEMA, + "local_static_contract_verified": contract["local_static_contract_verified"], + "local_gravitino_jdbc_catalog_restart_verified": ( + verified + and evidence is not None + and evidence.get("local_gravitino_jdbc_catalog_restart_verified") is True + ), + "local_authenticated_catalog_persistence_verified": ( + verified + and evidence is not None + and evidence.get("local_authenticated_catalog_persistence_verified") is True + ), + "persistent_catalog_identity_binding_verified": False, + "protected_workload_identity_verified": False, + "oidc_verified": False, + "tls_verified": False, + "spark_conformance_verified": False, + "flink_conformance_verified": False, + "production_ingestion_verified": False, + "production_ready": False, + "contract_fingerprint": contract["contract_fingerprint"], + "evidence_fingerprint": ( + evidence.get("evidence_fingerprint") if evidence else None + ), + "errors": errors, + } + + +def _write_json(path: Path, payload: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(payload, ensure_ascii=True, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + validate = subparsers.add_parser("validate") + validate.add_argument("--profile", type=Path, default=DEFAULT_PROFILE_PATH) + validate.add_argument("--evidence", type=Path, default=DEFAULT_EVIDENCE_PATH) + rehearse = subparsers.add_parser("rehearse") + rehearse.add_argument("--profile", type=Path, default=DEFAULT_PROFILE_PATH) + rehearse.add_argument("--evidence-out", type=Path, required=True) + verify = subparsers.add_parser("verify") + verify.add_argument("--evidence", type=Path, default=DEFAULT_EVIDENCE_PATH) + args = parser.parse_args(argv) + try: + if args.command == "validate": + report = build_validation_report( + profile_path=args.profile, evidence_path=args.evidence + ) + print(json.dumps(report, ensure_ascii=True, indent=2, sort_keys=True)) + return 0 if not report["errors"] else 1 + if args.command == "verify": + value = json.loads(args.evidence.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError("Gravitino JDBC restart evidence must be an object") + errors = verify_evidence_integrity(value) + print(json.dumps({"verified": not errors, "errors": errors}, indent=2)) + return 0 if not errors else 1 + evidence = run_live_rehearsal(args.profile) + _write_json(args.evidence_out, evidence) + print(json.dumps(evidence, ensure_ascii=True, indent=2, sort_keys=True)) + return 0 if not evidence["errors"] else 1 + except ( + KeyError, + OSError, + TypeError, + ValueError, + json.JSONDecodeError, + identity.MetadataFabricGravitinoIdentityError, + MetadataFabricGravitinoJdbcRestartError, + KeyboardInterrupt, + ) as exc: + print(f"metadata fabric Gravitino JDBC restart: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/data_agent/platform_truth.py b/data_agent/platform_truth.py index 673ce5fb..1f63d3ef 100644 --- a/data_agent/platform_truth.py +++ b/data_agent/platform_truth.py @@ -648,6 +648,26 @@ def _config( ), "Protected OIDC workload identity, TLS and production Gravitino catalog", ), + RuntimeSpec( + "metadata_gravitino_jdbc_restart_rehearsal", + "gravitino_catalog_restart_rehearsal", + "governed", + "evidence_durable", + "committed local authenticated JDBC catalog restart evidence", + "metadata-platform", + "local_verification_only", + ( + "data_agent/metadata_fabric_gravitino_jdbc_restart.py", + "scripts/metadata-fabric-gravitino-jdbc-restart.sh", + ), + ( + ( + "data_agent/metadata_fabric_gravitino_jdbc_restart.py", + "subprocess.run", + ), + ), + "Protected identity and production catalog durability/conformance gate", + ), RuntimeSpec( "datalake_monitor", "monitor_loop", diff --git a/data_agent/test_metadata_fabric_gravitino_jdbc_restart.py b/data_agent/test_metadata_fabric_gravitino_jdbc_restart.py new file mode 100644 index 00000000..ded2aa6b --- /dev/null +++ b/data_agent/test_metadata_fabric_gravitino_jdbc_restart.py @@ -0,0 +1,294 @@ +import json +from copy import deepcopy +from pathlib import Path + +import pytest +import yaml + +from data_agent import metadata_fabric_gravitino_jdbc_restart as restart + + +def _checked_evidence() -> dict: + return json.loads(restart.DEFAULT_EVIDENCE_PATH.read_text(encoding="utf-8")) + + +def _observation() -> dict: + return deepcopy(_checked_evidence()["observation"]) + + +def _write_profile(tmp_path: Path, value: dict) -> Path: + path = tmp_path / "profile.yaml" + path.write_text(yaml.safe_dump(value, sort_keys=False), encoding="utf-8") + return path + + +def test_checked_in_contract_and_evidence_verify_the_local_boundary(): + contract = restart.build_contract_report() + validation = restart.build_validation_report() + evidence = _checked_evidence() + + assert contract["contract_fingerprint"] == ( + "f622d8a61bae49171bc76a16bfe64280c616c028bddf88479f1ad04acb1dadf0" + ) + assert contract["local_static_contract_verified"] is True + assert contract["runtime_image_identity"] == { + "gravitino_host_image_id": restart.GRAVITINO_HOST_IMAGE_ID, + "gravitino_kubernetes_image_id": restart.GRAVITINO_KUBERNETES_IMAGE_ID, + "postgresql_image_digest": restart.POSTGRESQL_IMAGE_DIGEST, + } + assert restart.verify_evidence_integrity(evidence) == [] + assert validation["errors"] == [] + assert validation["local_gravitino_jdbc_catalog_restart_verified"] is True + assert validation["local_authenticated_catalog_persistence_verified"] is True + assert validation["persistent_catalog_identity_binding_verified"] is False + assert validation["protected_workload_identity_verified"] is False + assert validation["oidc_verified"] is False + assert validation["tls_verified"] is False + assert validation["production_ingestion_verified"] is False + assert validation["production_ready"] is False + + +def test_profile_rejects_privilege_expansion_and_sensitive_fields(tmp_path): + profile = yaml.safe_load(restart.DEFAULT_PROFILE_PATH.read_text(encoding="utf-8")) + profile["scope"]["role_securable_objects"][1]["privileges"].append( + {"name": "MODIFY_TABLE", "condition": "ALLOW"} + ) + profile["catalog"]["jdbc_password"] = "must-not-enter-profile" + + with pytest.raises( + restart.MetadataFabricGravitinoJdbcRestartError, + match="profile is invalid", + ): + restart.load_profile(_write_profile(tmp_path, profile)) + + +def test_profile_rejects_tampered_identity_dependency(tmp_path, monkeypatch): + profile = yaml.safe_load(restart.DEFAULT_PROFILE_PATH.read_text(encoding="utf-8")) + dependency = json.loads( + ( + restart.REPO_ROOT + / "docs/evidence/metadata-fabric-gravitino-identity-2026-07-28.json" + ).read_text(encoding="utf-8") + ) + dependency["production_identity_verified"] = True + dependency_path = ( + tmp_path + / "docs/evidence/metadata-fabric-gravitino-identity-2026-07-28.json" + ) + dependency_path.parent.mkdir(parents=True) + dependency_path.write_text(json.dumps(dependency), encoding="utf-8") + monkeypatch.setattr(restart, "REPO_ROOT", tmp_path) + + with pytest.raises( + restart.MetadataFabricGravitinoJdbcRestartError, + match="identity dependency does not match", + ): + restart.load_profile(_write_profile(tmp_path, profile)) + + +def test_manifest_rejects_committed_secret_and_incomplete_runtime( + tmp_path, monkeypatch +): + (tmp_path / "runtime.yaml").write_text( + """ +apiVersion: v1 +kind: Namespace +metadata: + name: gda-metadata-catalog-persistence +--- +apiVersion: v1 +kind: Secret +metadata: + name: forbidden +stringData: + password: forbidden +""".strip(), + encoding="utf-8", + ) + monkeypatch.setattr(restart, "MANIFEST_DIR", tmp_path) + + errors = restart._validate_manifest() + + assert "Gravitino JDBC restart manifest may not commit Secret values" in errors + assert "Gravitino JDBC restart manifest is incomplete" in errors + + +def test_manifest_requires_driver_init_container_resources(monkeypatch): + documents = restart._manifest_documents() + workload = next( + document + for document in documents + if document.get("kind") == "StatefulSet" + and document["metadata"]["name"] == "gravitino-persistence" + ) + driver_init = next( + item + for item in workload["spec"]["template"]["spec"]["initContainers"] + if item["name"] == "stage-postgresql-jdbc-driver" + ) + driver_init.pop("resources") + monkeypatch.setattr(restart, "_manifest_documents", lambda: documents) + + assert "Gravitino JDBC driver initContainer resources are incomplete" in ( + restart._validate_manifest() + ) + + +@pytest.mark.parametrize( + ("mutate", "expected"), + [ + ( + lambda value: value["restart"]["after"]["gravitino"].update( + {"pod_uid": value["restart"]["before"]["gravitino"]["pod_uid"]} + ), + "gravitino pod did not restart", + ), + ( + lambda value: value["restart"]["after"]["postgresql"]["pvc"].update( + {"uid": "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"} + ), + "postgresql PVC identity changed", + ), + ( + lambda value: value["restart"]["after"]["gravitino"].update( + {"image_id": "sha256:" + "0" * 64} + ), + "gravitino runtime image ID does not match", + ), + ( + lambda value: value["restart"]["after"]["postgresql"].update( + {"ready_replicas": 0} + ), + "postgresql was not ready around restart", + ), + ( + lambda value: value["restart"]["before"]["gravitino"].update( + {"image_id": "sha256:" + "0" * 64} + ), + "gravitino runtime image ID does not match", + ), + ( + lambda value: value["restart"]["before"]["postgresql"].update( + {"service_account": "default"} + ), + "postgresql service account token isolation failed", + ), + ( + lambda value: value["restart"]["before"].update( + {"gravitino_host_image_id": "sha256:" + "0" * 64} + ), + "before runtime boundary does not match", + ), + ], +) +def test_evidence_rejects_restart_or_pvc_identity_drift(mutate, expected): + observation = _observation() + mutate(observation) + + evidence = restart.build_evidence(observation) + + assert expected in evidence["errors"] + assert evidence["local_gravitino_jdbc_catalog_restart_verified"] is False + assert evidence["local_postgresql_pvc_restart_verified"] is False + assert evidence["local_warehouse_pvc_restart_verified"] is False + + +@pytest.mark.parametrize( + ("mutate", "expected"), + [ + ( + lambda value: value["post_restart"]["table"].update( + {"fingerprint": "0" * 64} + ), + "JDBC-backed table did not survive restart", + ), + ( + lambda value: value["post_restart"]["role"]["securable_objects"][1][ + "privileges" + ].append({"name": "MODIFY_TABLE", "condition": "ALLOW"}), + "minimum-privilege role did not survive restart", + ), + ( + lambda value: value["post_restart"].update( + {"denied_catalog_create_status": 200} + ), + "administrative catalog mutation was not denied", + ), + ( + lambda value: value["post_restart"]["authentication"].update( + {"bounded_status": 401} + ), + "authenticated principals did not survive restart", + ), + ], +) +def test_evidence_rejects_table_role_denial_or_authentication_drift( + mutate, expected +): + observation = _observation() + mutate(observation) + + evidence = restart.build_evidence(observation) + + assert expected in evidence["errors"] + assert evidence["local_authenticated_catalog_persistence_verified"] is False + + +def test_evidence_recomputes_table_fingerprint_instead_of_trusting_continuity(): + observation = _observation() + forged_projection = deepcopy( + observation["pre_restart"]["table"]["projection"] + ) + forged_projection["columns"][0]["nullable"] = True + forged_fingerprint = restart.recovery._canonical_sha256(forged_projection) + for phase in ("pre_restart", "post_restart"): + observation[phase]["table"]["projection"] = deepcopy(forged_projection) + observation[phase]["table"]["fingerprint"] = forged_fingerprint + + evidence = restart.build_evidence(observation) + + assert "JDBC-backed table did not survive restart" in evidence["errors"] + + +def test_evidence_rejects_incomplete_cleanup_and_sensitive_material(): + observation = _observation() + observation["runtime_checks"]["namespace_absent"] = False + observation["database_password"] = "must-not-enter-evidence" + + evidence = restart.build_evidence(observation) + + assert "Gravitino JDBC restart observation contains sensitive material" in evidence[ + "errors" + ] + assert "Gravitino JDBC restart cleanup is incomplete" in evidence["errors"] + assert evidence["local_gravitino_jdbc_catalog_restart_verified"] is False + + +def test_evidence_integrity_rejects_tampering_and_production_overclaim(): + evidence = _checked_evidence() + evidence["observation"]["post_restart"]["table"]["name"] = "tampered" + + errors = restart.verify_evidence_integrity(evidence) + + assert "Gravitino JDBC restart evidence fingerprint does not match" in errors + + forged = _checked_evidence() + forged["persistent_catalog_identity_binding_verified"] = True + forged["production_ready"] = True + stable = { + key: value for key, value in forged.items() if key != "evidence_fingerprint" + } + forged["evidence_fingerprint"] = restart.recovery._canonical_sha256(stable) + errors = restart.verify_evidence_integrity(forged) + assert ( + "Gravitino JDBC restart evidence may not claim " + "persistent_catalog_identity_binding_verified" + ) in errors + assert "Gravitino JDBC restart evidence may not claim production_ready" in errors + + +def test_wrapper_is_fail_closed(): + wrapper = restart.DEFAULT_WRAPPER_PATH.read_text(encoding="utf-8") + + assert "set -euo pipefail" in wrapper + assert "metadata_fabric_gravitino_jdbc_restart" in wrapper diff --git a/data_agent/test_platform_truth.py b/data_agent/test_platform_truth.py index 6f5e03fe..a7e33ab1 100644 --- a/data_agent/test_platform_truth.py +++ b/data_agent/test_platform_truth.py @@ -213,6 +213,11 @@ def test_repository_source_access_and_runtime_baselines_match(): and item["production_role"] == "local_verification_only" for item in static_report["runtime"]["inventory"] ) + assert any( + item["runtime_id"] == "metadata_gravitino_jdbc_restart_rehearsal" + and item["production_role"] == "local_verification_only" + for item in static_report["runtime"]["inventory"] + ) def test_runtime_report_detects_unregistered_background_mechanism(tmp_path): diff --git a/docs/architecture-decisions/adr-054-local-gravitino-jdbc-catalog-restart-continuity.md b/docs/architecture-decisions/adr-054-local-gravitino-jdbc-catalog-restart-continuity.md new file mode 100644 index 00000000..099ac348 --- /dev/null +++ b/docs/architecture-decisions/adr-054-local-gravitino-jdbc-catalog-restart-continuity.md @@ -0,0 +1,68 @@ +# ADR-054: Local Gravitino JDBC Catalog Restart Continuity + +**Status**: Accepted + +**Date**: 2026-07-29 + +**Decision owners**: Metadata Platform, Data Governance, SRE, Platform Architecture + +**Related decisions**: [ADR-037](adr-037-local-metadata-fabric-foundation-sandbox.md) · [ADR-052](adr-052-local-gravitino-basic-bounded-provider-identity.md) · [ADR-053](adr-053-production-metadata-fabric-identity-readiness-gate.md) + +## Context + +M3-6 proved that a Gravitino `1.3.0` Basic user can be limited to table creation while catalog creation is denied. Its memory-backed catalog and ephemeral database could not prove that the same authenticated identity, authorization objects and table metadata remain usable after a provider or catalog database restart. M3-7 therefore kept `persistent_catalog_restart` as a required production attestation check. + +The next bounded step is a real local persistence rehearsal, not another self-asserted production gate. It must reuse the exact M3-6 privilege boundary, place both the Iceberg catalog metadata and warehouse on persistent volumes, restart both stateful providers and read the same table afterward. It still cannot prove production OIDC, TLS, storage durability, protected workload identity or engine conformance. + +## Decision + +### 1. Isolate an authenticated persistent catalog rehearsal + +The rehearsal creates only the temporary `gda-metadata-catalog-persistence` namespace in Docker Desktop Kubernetes. PostgreSQL `16.10-bookworm` stores both the Gravitino relational entity schema and a separate `iceberg` catalog database on one `standard` PVC. Gravitino stores the local Iceberg warehouse on a second `standard` PVC. Both workloads use StatefulSets, ClusterIP Services and dedicated ServiceAccounts with token automount disabled. + +No Secret is committed. The administrator, bounded-user and database materials are generated for one run, projected through an ephemeral Kubernetes Secret and excluded from observations, errors and evidence. The namespace and both dynamically provisioned volumes must be deleted before evidence can pass. + +### 2. Use the real Iceberg JDBC backend and isolated plugin driver + +The catalog provider is `lakehouse-iceberg`, `catalog-backend=jdbc`, with URI `jdbc:postgresql://gravitino-persistence-postgresql:5432/iceberg` and warehouse `file:///var/lib/gravitino/warehouse`. An initContainer copies the PostgreSQL driver from the versioned local Gravitino image into the Iceberg plugin's isolated classloader path; the runtime must prove that this mounted driver is readable. + +The checked profile distinguishes the Docker host image ID (`d355dc7e...`) from the Kubernetes node runtime image ID (`18e24b43...`). Docker Desktop's local kind node reports the latter in Pod status even though host `docker image inspect` reports the former. Treating both fields as one image digest caused the first otherwise-successful rehearsal to fail closed, so each identity now has an explicit name and contract field. + +### 3. Preserve the M3-6 authorization boundary across restart + +The Basic service administrator provisions metalake `gda_persistence`, catalog `lakehouse`, schema `published`, the bounded user and role. The bounded role contains exactly: + +- `USE_CATALOG` on `lakehouse`; +- `USE_SCHEMA` and `CREATE_TABLE` on `lakehouse.published`. + +Before restart, the bounded user must authenticate, create/read `gda_persistence_probe`, and receive 403 when attempting to create `unauthorized_catalog`. The rehearsal then restarts PostgreSQL followed by Gravitino. After restart, it creates a new authenticated client using the same bounded identity, reads the same table projection and role, and must again receive 403 for catalog creation. + +### 4. Require stable stateful identity and changed process identity + +For both workloads, the StatefulSet UID and PVC UID must remain unchanged while the Pod UID must change. Each PVC must remain `Bound` on `standard`, each StatefulSet must return to one ready replica, the expected runtime image identity must match, and the table projection fingerprint must be byte-for-byte identical before and after restart. + +Evidence may set only four local claims to true: `local_gravitino_jdbc_catalog_restart_verified`, `local_authenticated_catalog_persistence_verified`, `local_postgresql_pvc_restart_verified` and `local_warehouse_pvc_restart_verified`. `persistent_catalog_identity_binding_verified`, `protected_workload_identity_verified`, `oidc_verified`, `tls_verified`, Spark/Flink conformance, production ingestion and `production_ready` remain false. + +## Verification + +The final Docker Desktop rehearsal on 2026-07-29 produced contract fingerprint `f622d8a61bae49171bc76a16bfe64280c616c028bddf88479f1ad04acb1dadf0` and evidence fingerprint `34792bb47ad71041a87adeb644439bf9b6aa3f4855cdc98782d6e3b4282bf1aa`. + +- PostgreSQL and Gravitino Pod UIDs changed while both StatefulSet UIDs and PVC UIDs remained stable; +- both PVCs remained `Bound` on `standard`, and both workloads returned to one ready replica; +- the bounded principal authenticated before and after restart, while catalog creation returned 403 both times; +- the exact role read back before and after restart without privilege expansion; +- table create/read returned 200 before restart and read returned 200 afterward; +- table projection fingerprint `25845cd3890a9e4dc1663cf9e77bfe6a63144223d29ab788f12af6448d0717a3` remained identical; +- both port-forwards stopped, the namespace was deleted, and no provider object or persistent volume was retained. + +Focused tests cover profile, dependency and manifest validation; Pod replacement; StatefulSet/PVC continuity; runtime image identity; table, role, authentication and denial continuity; cleanup; sensitive-field rejection; evidence tampering; and production overclaim. Required CI validates the checked evidence and runs the focused module. The live Kubernetes rehearsal remains an explicit local operator action. + +## Consequences + +**Positive**: the Gravitino path now has tangible, authenticated Iceberg JDBC persistence evidence instead of a memory-catalog assumption. Restart continuity is bound to both storage identity and the exact minimum-privilege role. + +**Negative**: the database and warehouse are two local Docker Desktop PVCs in one cluster and failure domain. Basic credentials, loopback HTTP and `file://` warehouse storage remain local-only mechanisms. The rehearsal deletes its volumes, so it proves controlled restart continuity rather than backup recovery or long-duration durability. + +**Mitigation**: keep all protected identity, OIDC, TLS, cross-cluster storage, production ingestion and conformance claims false. Production acceptance still requires a selected protected authentication path, immutable registry provenance, object-store warehouse, managed catalog database, backup/PITR, tenant isolation, Spark/Flink interoperability and protected-environment attestation. + +**Revisit trigger**: Gravitino version, Iceberg catalog implementation, JDBC schema, driver classloading, storage class, production identity architecture or catalog durability target changes. diff --git a/docs/evidence/metadata-fabric-gravitino-jdbc-restart-2026-07-29.json b/docs/evidence/metadata-fabric-gravitino-jdbc-restart-2026-07-29.json new file mode 100644 index 00000000..4aa826f6 --- /dev/null +++ b/docs/evidence/metadata-fabric-gravitino-jdbc-restart-2026-07-29.json @@ -0,0 +1,261 @@ +{ + "catalog_backend": "jdbc", + "environment": "local_docker_desktop", + "errors": [], + "evidence_fingerprint": "34792bb47ad71041a87adeb644439bf9b6aa3f4855cdc98782d6e3b4282bf1aa", + "flink_conformance_verified": false, + "identity_evidence_fingerprint": "f0b0de1f80f079d43318937e0a0cc151a8546e9e307bef204738b1367f9b29fd", + "local_authenticated_catalog_persistence_verified": true, + "local_gravitino_jdbc_catalog_restart_verified": true, + "local_postgresql_pvc_restart_verified": true, + "local_warehouse_pvc_restart_verified": true, + "observation": { + "contract": { + "contract_fingerprint": "f622d8a61bae49171bc76a16bfe64280c616c028bddf88479f1ad04acb1dadf0", + "identity_evidence_fingerprint": "f0b0de1f80f079d43318937e0a0cc151a8546e9e307bef204738b1367f9b29fd", + "local_static_contract_verified": true + }, + "observed_at": "2026-07-28T23:57:52.919163+00:00", + "post_restart": { + "authentication": { + "admin_status": 200, + "bounded_status": 200, + "material_recorded": false + }, + "denied_catalog_create_status": 403, + "role": { + "name": "gda-table-projection", + "securable_objects": [ + { + "fullName": "lakehouse", + "privileges": [ + { + "condition": "ALLOW", + "name": "USE_CATALOG" + } + ], + "type": "CATALOG" + }, + { + "fullName": "lakehouse.published", + "privileges": [ + { + "condition": "ALLOW", + "name": "CREATE_TABLE" + }, + { + "condition": "ALLOW", + "name": "USE_SCHEMA" + } + ], + "type": "SCHEMA" + } + ] + }, + "table": { + "fingerprint": "25845cd3890a9e4dc1663cf9e77bfe6a63144223d29ab788f12af6448d0717a3", + "name": "gda_persistence_probe", + "projection": { + "columns": [ + { + "name": "probe_id", + "nullable": false, + "type": "string" + } + ], + "name": "gda_persistence_probe", + "probe_property": "true" + }, + "read_status": 200 + } + }, + "pre_restart": { + "authentication": { + "admin_status": 200, + "bounded_status": 200, + "material_recorded": false + }, + "catalog": { + "backend": "jdbc", + "catalog": "lakehouse", + "jdbc_initialize": true, + "material_recorded": false, + "metalake": "gda_persistence", + "provider": "lakehouse-iceberg", + "schema": "published", + "uri": "jdbc:postgresql://gravitino-persistence-postgresql:5432/iceberg", + "warehouse": "file:///var/lib/gravitino/warehouse" + }, + "denied_catalog_create_status": 403, + "role": { + "name": "gda-table-projection", + "securable_objects": [ + { + "fullName": "lakehouse", + "privileges": [ + { + "condition": "ALLOW", + "name": "USE_CATALOG" + } + ], + "type": "CATALOG" + }, + { + "fullName": "lakehouse.published", + "privileges": [ + { + "condition": "ALLOW", + "name": "CREATE_TABLE" + }, + { + "condition": "ALLOW", + "name": "USE_SCHEMA" + } + ], + "type": "SCHEMA" + } + ] + }, + "table": { + "create_status": 200, + "fingerprint": "25845cd3890a9e4dc1663cf9e77bfe6a63144223d29ab788f12af6448d0717a3", + "name": "gda_persistence_probe", + "projection": { + "columns": [ + { + "name": "probe_id", + "nullable": false, + "type": "string" + } + ], + "name": "gda_persistence_probe", + "probe_property": "true" + }, + "read_status": 200 + } + }, + "restart": { + "after": { + "context": "docker-desktop", + "gravitino": { + "image": "docker.io/gda/gravitino:1.3.0-local-arm64", + "image_id": "sha256:18e24b43be854dabdc13e96b1019eb3dc691d59cc64e411aa6a3cc49225fe2d3", + "node_name": "desktop-worker", + "pod_name": "gravitino-persistence-0", + "pod_uid": "0882bb9a-19a2-44c2-98ec-eeca241ad323", + "pvc": { + "name": "warehouse-gravitino-persistence-0", + "phase": "Bound", + "storage_class": "standard", + "uid": "3149a94a-1f97-46c5-9d56-07f08a652157", + "volume_name": "pvc-3149a94a-1f97-46c5-9d56-07f08a652157" + }, + "ready_replicas": 1, + "service_account": "gravitino-persistence", + "service_account_automount_disabled": true, + "statefulset_uid": "0192c1e2-6abb-4c75-8bb6-84a6887244e2" + }, + "gravitino_host_image_id": "sha256:d355dc7e92f9e3545d717f3eab2cbdf412115f2b82e1e544d7f6235c1eacd5a5", + "jdbc_driver_mounted": true, + "namespace": { + "name": "gda-metadata-catalog-persistence", + "uid": "1488b7bb-0243-42a4-affd-8c824da66dd2" + }, + "postgresql": { + "image": "docker.io/library/postgres:16.10-bookworm", + "image_id": "docker.io/library/postgres@sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74", + "node_name": "desktop-worker", + "pod_name": "gravitino-persistence-postgresql-0", + "pod_uid": "0cd6c611-6235-403a-adf6-941ee5b40644", + "pvc": { + "name": "data-gravitino-persistence-postgresql-0", + "phase": "Bound", + "storage_class": "standard", + "uid": "7f9d008a-f117-41da-9edd-2240a7e9b82e", + "volume_name": "pvc-7f9d008a-f117-41da-9edd-2240a7e9b82e" + }, + "ready_replicas": 1, + "service_account": "gravitino-persistence-postgresql", + "service_account_automount_disabled": true, + "statefulset_uid": "c97439ca-9d3a-43cc-915e-3672cac5556e" + }, + "service": { + "name": "gravitino-persistence", + "type": "ClusterIP", + "uid": "4a13ca72-cf65-499b-ac8a-397b16b3849e" + }, + "source_schema_sha256": "7a2d605a677a462ca619dba594ce7ebcf500358345560ad084c1b67a25c722df" + }, + "before": { + "context": "docker-desktop", + "gravitino": { + "image": "docker.io/gda/gravitino:1.3.0-local-arm64", + "image_id": "sha256:18e24b43be854dabdc13e96b1019eb3dc691d59cc64e411aa6a3cc49225fe2d3", + "node_name": "desktop-worker", + "pod_name": "gravitino-persistence-0", + "pod_uid": "9b1f3b78-fcd1-4b63-9a71-6bc80ee4e219", + "pvc": { + "name": "warehouse-gravitino-persistence-0", + "phase": "Bound", + "storage_class": "standard", + "uid": "3149a94a-1f97-46c5-9d56-07f08a652157", + "volume_name": "pvc-3149a94a-1f97-46c5-9d56-07f08a652157" + }, + "ready_replicas": 1, + "service_account": "gravitino-persistence", + "service_account_automount_disabled": true, + "statefulset_uid": "0192c1e2-6abb-4c75-8bb6-84a6887244e2" + }, + "gravitino_host_image_id": "sha256:d355dc7e92f9e3545d717f3eab2cbdf412115f2b82e1e544d7f6235c1eacd5a5", + "jdbc_driver_mounted": true, + "namespace": { + "name": "gda-metadata-catalog-persistence", + "uid": "1488b7bb-0243-42a4-affd-8c824da66dd2" + }, + "postgresql": { + "image": "docker.io/library/postgres:16.10-bookworm", + "image_id": "docker.io/library/postgres@sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74", + "node_name": "desktop-worker", + "pod_name": "gravitino-persistence-postgresql-0", + "pod_uid": "b22cf193-9a78-44a9-940b-bc734019ed4f", + "pvc": { + "name": "data-gravitino-persistence-postgresql-0", + "phase": "Bound", + "storage_class": "standard", + "uid": "7f9d008a-f117-41da-9edd-2240a7e9b82e", + "volume_name": "pvc-7f9d008a-f117-41da-9edd-2240a7e9b82e" + }, + "ready_replicas": 1, + "service_account": "gravitino-persistence-postgresql", + "service_account_automount_disabled": true, + "statefulset_uid": "c97439ca-9d3a-43cc-915e-3672cac5556e" + }, + "service": { + "name": "gravitino-persistence", + "type": "ClusterIP", + "uid": "4a13ca72-cf65-499b-ac8a-397b16b3849e" + }, + "source_schema_sha256": "7a2d605a677a462ca619dba594ce7ebcf500358345560ad084c1b67a25c722df" + } + }, + "runtime_checks": { + "all_port_forwards_stopped": true, + "kubernetes_service_account_used_for_provider_login": false, + "material_recorded": false, + "namespace_absent": true, + "namespace_delete_completed": true, + "persistent_volumes_retained": false, + "provider_objects_retained": false + }, + "schema": "gda.metadata_fabric_gravitino_jdbc_restart_observation.v1" + }, + "oidc_verified": false, + "persistent_catalog_identity_binding_verified": false, + "production_ingestion_verified": false, + "production_ready": false, + "protected_workload_identity_verified": false, + "restart_scope": "postgresql_then_gravitino", + "schema": "gda.metadata_fabric_gravitino_jdbc_restart_evidence.v1", + "spark_conformance_verified": false, + "tls_verified": false +} diff --git a/docs/roadmap-ar0-platform-truth-2026-07-24.md b/docs/roadmap-ar0-platform-truth-2026-07-24.md index 6bc69e1d..4527d915 100644 --- a/docs/roadmap-ar0-platform-truth-2026-07-24.md +++ b/docs/roadmap-ar0-platform-truth-2026-07-24.md @@ -199,7 +199,7 @@ Temporal 继续保持目标组件状态，不在这一包并行接入。OpenMeta 当前完成仅指本地合同、授权 evidence、outbox/callback 代码、数据库成功终局门、托管 worker 代码、默认关闭的部署模板及离线 activation/release preflight、candidate/registry/provenance/artifact-release/live observation evidence gate、合成 golden slice、定向测试、真实 PostgreSQL 16 事务边界和 canonical mainline 治理。`candidate_validated`、`registry_subject_bound`、本地合成 `provenance_verified`、`ready_for_activation`、`ready_for_staging_apply`、`verified_for_staging_apply` 和本地 live collection 都不等于真实镜像已 attested 或 staging 已部署；真实 IAM/OIDC 与 service token 生命周期、首次 GHCR publish/verify、真实 provenance artifact verify、registry-backed live staging revision、worker/callback 扩容运行、golden slice staging 运行链、受保护 release/live evidence provenance、独立 DolphinScheduler metadata PostgreSQL 和真实数据终局证据仍属于 4.7 后续切片。 -### 4.8 Metadata Fabric Bridge M1 + M2 + M3-7（生产双 provider 身份验收合同已冻结） +### 4.8 Metadata Fabric Bridge M1 + M2 + M3-8（本地认证 JDBC catalog 重启连续性已验证） 第八块回到 AR-1 的 metadata control plane，以 [ADR-036](architecture-decisions/adr-036-read-only-metadata-fabric-bridge-contract.md) 固定 OpenMetadata + Gravitino + GDA Control Ledger 的首条 table slice： @@ -227,8 +227,9 @@ Temporal 继续保持目标组件状态，不在这一包并行接入。OpenMeta 22. [ADR-051](architecture-decisions/adr-051-local-openmetadata-bounded-provider-identity.md) 已在 OpenMetadata `1.13.1` 创建临时非管理员 bot，effective roles 精确为 provider 强制 `DefaultBotRole` 加项目专用 role；项目新增 policy 仅允许 `table/Create`。bot 创建/read-back 临时 table 分别返回 201/200，创建 policy 返回 403；JWT 轮换后旧值返回 401，新值返回 200，吊销后新值返回 401。table、bot、user、role、policy 与 denial probe 最终均不存在，evidence SHA 为 `61b6a3429ae948f563bfc2bd012d8b586be581704cec646fd5e74b991243f03f`。该 scoped local 结论不包含 Gravitino、OIDC、Kubernetes-to-provider identity exchange 或生产 credential delivery。 23. [ADR-052](architecture-decisions/adr-052-local-gravitino-basic-bounded-provider-identity.md) 已在隔离 Gravitino `1.3.0` namespace 启用 Basic IdP 与 authorization，bounded user 只获得 `lakehouse` 的 `USE_CATALOG` 以及 `lakehouse.published` 的 `USE_SCHEMA`/`CREATE_TABLE`；table create/read 为 200/200，越权 catalog create 为 403，密码轮换使旧值返回 401，IdP 用户删除使替换值返回 401。临时 metalake/catalog/schema/table/user/role、namespace 与 loopback port-forward 均已清理，证据 SHA 为 `f0b0de1f80f079d43318937e0a0cc151a8546e9e307bef204738b1367f9b29fd`。这只是 Gravitino Basic local POC；probe catalog 使用 memory backend，不包含 OIDC、TLS、Kubernetes-to-provider identity exchange、持久生产 catalog 或双 provider production identity。 24. [ADR-053](architecture-decisions/adr-053-production-metadata-fabric-identity-readiness-gate.md) 已将 OIDC federation、双 provider integration、digest-pinned authentication component、workload/tenant claim、Kubernetes ServiceAccount、禁止 direct bypass、short-lived token、M3-5/M3-6 精确 allow/deny contract、TLS/mTLS、持久 Gravitino catalog、tenant isolation、owner/audit/SLO/runbook 与 18 项 protected attestation check 冻结为 fail-closed profile。Gravitino `1.3.0` 镜像只发现 Basic IdP jar，因此生产只允许明确选择并证明 `custom_oidc_authenticator` 或 `identity_aware_proxy`，不假设 native OIDC。当前 profile fingerprint 为 `2e9d5cac3560b853820f923669f6794ead63bcb36a528639fc0e9539e148ee2f`，合同有效但 40 项外部输入 blocked，没有真实 attestation，全部 production identity claims 仍为 `false`。 +25. [ADR-054](architecture-decisions/adr-054-local-gravitino-jdbc-catalog-restart-continuity.md) 已在隔离 namespace 中将 Gravitino Iceberg catalog metadata 落到 PostgreSQL JDBC、warehouse 落到独立 PVC，并复用 M3-6 精确 `USE_CATALOG`/`USE_SCHEMA`/`CREATE_TABLE` 角色。依次重启 PostgreSQL 与 Gravitino 后，两者 Pod UID 均变化而 StatefulSet/PVC UID 保持；同一 bounded user 重新认证并读取相同 table fingerprint，catalog create 前后均为 403，namespace/PV 完整清理。evidence fingerprint 为 `34792bb47ad71041a87adeb644439bf9b6aa3f4855cdc98782d6e3b4282bf1aa`。该结果仅证明 Docker Desktop 单集群、本地 Basic/HTTP/file warehouse 的 restart continuity，不等于 production persistent identity binding、OIDC、TLS、备份恢复、Spark/Flink conformance 或生产 ingestion。 -此处 M1 只证明静态合同和只读 HTTP 边界；M2a 只证明本地 live foundation 与 PVC 重挂载连续性；M2b-1/M2b-2 分别限定在同集群新 PVC 和同集群隔离 repository；M2b-3 的 `local_cross_cluster_recovery_verified=true` 只限定在 `local_same_host_distinct_kubernetes_clusters_external_s3_repository`；M2c-1/M2c-2/M2c-3 分别限定本地 provider metrics、临时双周期 OTel 和单 job scrape recovery；M2c-4/M2d-2 只证明 production observability/NetworkPolicy profile 与 attestation 合同可校验；M2d-1 只证明本地两节点 kindnet 的隔离合成流量；M3-1 的 terminal evidence 与 M3-2 的 PolicyDecision/Approval 仍是 deterministic local fixtures。M3-2 只把 projection 写入本地 provider 并证明 retained target 的单次零写入 replay；M3-3 只把该本地 evidence 对应的 binding 写入临时 GDA Control 账本；M3-4 只向无认证 loopback receiver 发送精确 candidate 并验证 503 后幂等恢复；M3-5 只证明 OpenMetadata 在 provider 强制默认 role 之上的项目新增 grant 限定为 `table/Create`，以及本地 JWT 轮换/吊销和越权拒绝；M3-6 只证明隔离 Gravitino Basic IdP 的 bounded table-create、catalog-create 拒绝、登录轮换/吊销和完整清理；M3-7 只证明 pending production identity profile、profile-bound attestation 和派生 claim 的 fail-closed 合同可校验，没有部署或证明真实身份路径。M3-2 ingestion 仍使用 bootstrap admin，Gravitino M3-6 probe catalog 使用 memory backend，restart persistence 未验证。生产持久 binding、ResourceVersion 和 legacy authority 都未写入；双 provider/生产最小权限、protected workload identity、OIDC、TLS、持久 catalog、tenant isolation、真实 receiver/alert/SLO、受保护 provider policy、生产 OpenLineage、生产 ingest/conformance、三项 production gate 和 `production_ready` 仍为 `false`。 +此处 M1 只证明静态合同和只读 HTTP 边界；M2a 只证明本地 live foundation 与 PVC 重挂载连续性；M2b-1/M2b-2 分别限定在同集群新 PVC 和同集群隔离 repository；M2b-3 的 `local_cross_cluster_recovery_verified=true` 只限定在 `local_same_host_distinct_kubernetes_clusters_external_s3_repository`；M2c-1/M2c-2/M2c-3 分别限定本地 provider metrics、临时双周期 OTel 和单 job scrape recovery；M2c-4/M2d-2 只证明 production observability/NetworkPolicy profile 与 attestation 合同可校验；M2d-1 只证明本地两节点 kindnet 的隔离合成流量；M3-1 的 terminal evidence 与 M3-2 的 PolicyDecision/Approval 仍是 deterministic local fixtures。M3-2 只把 projection 写入本地 provider 并证明 retained target 的单次零写入 replay；M3-3 只把该本地 evidence 对应的 binding 写入临时 GDA Control 账本；M3-4 只向无认证 loopback receiver 发送精确 candidate 并验证 503 后幂等恢复；M3-5 只证明 OpenMetadata 在 provider 强制默认 role 之上的项目新增 grant 限定为 `table/Create`，以及本地 JWT 轮换/吊销和越权拒绝；M3-6 只证明隔离 Gravitino Basic IdP 的 bounded table-create、catalog-create 拒绝、登录轮换/吊销和完整清理；M3-7 只证明 pending production identity profile、profile-bound attestation 和派生 claim 的 fail-closed 合同可校验，没有部署或证明真实身份路径；M3-8 只证明同一 Docker Desktop 集群内 Basic 用户、JDBC metadata 与 file warehouse PVC 在受控 Pod restart 后连续，不证明生产 failure domain、持久 identity binding 或 engine conformance。M3-2 ingestion 仍使用 bootstrap admin，生产持久 binding、ResourceVersion 和 legacy authority 都未写入；双 provider/生产最小权限、protected workload identity、OIDC、TLS、生产持久 catalog、tenant isolation、真实 receiver/alert/SLO、受保护 provider policy、生产 OpenLineage、生产 ingest/conformance、三项 production gate 和 `production_ready` 仍为 `false`。 ## 5. 重新评估条件 diff --git a/docs/system-of-record-matrix-2026-07-24.md b/docs/system-of-record-matrix-2026-07-24.md index 8180b267..8dcf150e 100644 --- a/docs/system-of-record-matrix-2026-07-24.md +++ b/docs/system-of-record-matrix-2026-07-24.md @@ -1,10 +1,10 @@ # GIS Data Agent System-of-Record 矩阵 -日期：2026-07-28 +日期：2026-07-29 -阶段：AR-0 `in_progress`；AR-1 gateway、成功终局 evidence gate、DolphinScheduler adapter sandbox POC、Metadata Fabric M1/M2、M2c-4/M2d-2 production readiness contracts、M3-1/M3-2、M3-3 local binding ledger、M3-4 local OpenLineage wire delivery、M3-5 local OpenMetadata bounded identity、M3-6 local Gravitino Basic bounded identity 与 M3-7 production identity readiness contract 已验证，生产 provider ingestion、生产观测、生产 policy/tenant isolation、生产 identity attestation 和生产切换仍 `in_progress` +阶段：AR-0 `in_progress`；AR-1 gateway、成功终局 evidence gate、DolphinScheduler adapter sandbox POC、Metadata Fabric M1/M2、M2c-4/M2d-2 production readiness contracts、M3-1/M3-2、M3-3 local binding ledger、M3-4 local OpenLineage wire delivery、M3-5 local OpenMetadata bounded identity、M3-6 local Gravitino Basic bounded identity、M3-7 production identity readiness contract 与 M3-8 local Gravitino JDBC restart continuity 已验证，生产 provider ingestion、生产观测、生产 policy/tenant isolation、生产 identity attestation 和生产切换仍 `in_progress` -适用分支：`feat/ar1-metadata-fabric-identity-readiness-gate` +适用分支：`feat/ar1-metadata-fabric-jdbc-catalog-restart` ## 判定规则 @@ -20,12 +20,12 @@ | SQL schema 历史 | PostgreSQL `schema_migrations`，以完整 migration ID + checksum 为权威 | migration CLI 的 JSON 报告 | 保持现有 ledger；任何 drift fail closed | Data Platform | AR-0，已验证 | | 部署配置策略 | Compose/K8s/进程环境；`platform_truth.CONFIG_SPECS` 定义关键类型与策略；DolphinScheduler worker 有默认零副本、外部 ConfigMap/Secret 驱动的 Kustomize 模板、静态 validator 和 staging activation preflight | `.env` 仅补默认；脱敏 snapshot、Secret key attestation、未扩容 Deployment 和 `ready_for_activation` 都是观测/模板 | 版本化 DeploymentProfile + secret reference；部署环境始终优先；模板或 preflight 通过都不等于环境已启用 | Platform/SRE/Security | AR-0，部分实现；worker 模板/preflight 本地已验证 | | 环境发布与晋级 | 本地 candidate/registry/provenance/release/live 合同已绑定 publisher、verifier、OCI 和 manifest identity；canonical `main@0182406`、archive refs、三组 active ruleset 与 `staging-provenance` protected environment 已建立，但尚无成功 publisher/verifier 或 deployment | 旧 mainline、feature branch、CI artifact、JSON、离线 report 和合成 `verified_for_staging_apply` 都不能单独成为发布权威；publisher SHA、verifier SHA 与 branch lineage 必须分别验证 | 由受保护 environment 的 DeploymentRevision 绑定 OCI、provenance artifact、release manifest 与全部 live verdict | Platform/SRE/Security/Repository Owner | AR-1 mainline 治理已恢复 -> 首次 GHCR publish/verify -> 真实 staging | -| 后台运行时清单 | `platform_truth.RUNTIME_INVENTORY` 是代码层登记；`gda_control` 已有受控 PlatformRun 写入口；DolphinScheduler managed worker 已登记但尚无生产调用方；M2b recovery runner、M2c-1 provider probe、M2c-2 `_OtelPortForward`、M2c-3 failure rehearsal 与 M2d-1 NetworkPolicy rehearsal 均登记为 `local_verification_only`，不是 scheduler、worker、持续监控、生产 policy controller 或状态权威 | AST primitive report、worker status JSON、FrameworkAttemptObservation、DolphinScheduler instance state、本地 recovery/metrics/network-policy evidence | PlatformRun ledger 唯一登记最终状态；framework/provider attempt 只能回报观测；本地演练进程与 evidence 不得变成生产控制器、监控后端或 tenant-isolation 权威 | Platform Architecture | AR-1 adapter/worker 本地已验证；metadata recovery/metrics/policy runner 仅本地验证 -> staging 控制链待接入 | +| 后台运行时清单 | `platform_truth.RUNTIME_INVENTORY` 是代码层登记；`gda_control` 已有受控 PlatformRun 写入口；DolphinScheduler managed worker 已登记但尚无生产调用方；M2b recovery runner、M2c-1 provider probe、M2c-2 `_OtelPortForward`、M2c-3 failure rehearsal、M2d-1 NetworkPolicy rehearsal 与 M3-8 JDBC restart runner 均登记为 `local_verification_only`，不是 scheduler、worker、持续监控、生产 policy/catalog controller 或状态权威 | AST primitive report、worker status JSON、FrameworkAttemptObservation、DolphinScheduler instance state、本地 recovery/metrics/network-policy/catalog-restart evidence | PlatformRun ledger 唯一登记最终状态；framework/provider attempt 只能回报观测；本地演练进程与 evidence 不得变成生产控制器、监控后端、catalog authority 或 tenant-isolation 权威 | Platform Architecture | AR-1 adapter/worker 本地已验证；metadata recovery/metrics/policy/catalog runner 仅本地验证 -> staging 控制链待接入 | | 原始文件/对象 | 当前 local uploads、S3/MinIO/OBS 均可能被直接写入，权威边界未统一 | 临时上传、下载缓存、预览文件 | Landing object 以 immutable URI + checksum + retention 为权威；本地 scratch 可删除 | Data Platform | AR-2 | | 湖仓表与 snapshot | Iceberg/STAC/S3A 有局部实现，尚无通用发布权威 | STAC item、GeoParquet export | Iceberg catalog snapshot 是分析表版本权威；对象是物理内容，STAC 是发现投影 | Data Platform | AR-2 | | 在线空间数据 | PostGIS 业务表是当前编辑/查询事实，部分临时表混入 | Martin MVT、API JSON、导出文件 | 已批准 DataProductVersion 物化到 PostGIS；不能由瓦片或临时表反向定义产品版本 | GIS/Data Platform | AR-2 -> AR-4 | | 数据资产身份与版本 | `gda_control.resource/resource_version` 已实现 identity、hash、predecessor、tenant FK 和幂等 gateway 写入；`agent_data_assets`、`agent_asset_versions` 仍是兼容写路径 | UI catalog、search index、STAC | GDA ledger 管身份与版本绑定；旧行只有在 tenant、authority identity、checksum 和 version evidence 完整时才可形成 eligible plan；OpenMetadata 管治理目录，Gravitino 管技术对象映射 | Metadata Platform | AR-1 gateway 已验证 -> 生产切换待验收 | -| 技术元数据 | M1 已冻结 Gravitino table ref/reconciliation；M2 已验证本地 foundation/recovery/metrics/policy 和 production readiness contracts；M3-1 固定 technical projection intent，M3-2 已在 Gravitino memory catalog 创建/read-back，M3-3 将验证后的 ref 追加到 tenant-scoped 本地 binding ledger；M3-6 又在隔离 Gravitino Basic IdP 中验证 bounded table-create、catalog-create 拒绝、密码轮换/用户吊销和完整清理；M3-7 已冻结 production identity profile/attestation gate | harvester 结果、合成 response、本地 sandbox/recovery/metrics/policy/ingestion/identity observation、projection plan、provider evidence、binding ledger 与 readiness report | 源系统技术对象是原始证据；Gravitino 映射并联邦，不能覆盖业务 ResourceVersion；GDA binding ledger 只记录已验证关系，本地 memory catalog/evidence 不得冒充生产持久技术权威；Basic IdP、loopback HTTP、pending profile 和合成 attestation 都不是生产身份 | Metadata Platform | AR-1 M1/M2 + M3-6 local identity + M3-7 readiness contract 已验证 -> 认证持久 catalog/受保护 attestation/生产 conformance 待执行 | +| 技术元数据 | M1 已冻结 Gravitino table ref/reconciliation；M2 已验证本地 foundation/recovery/metrics/policy 和 production readiness contracts；M3-1 固定 technical projection intent，M3-2 已在 Gravitino memory catalog 创建/read-back，M3-3 将验证后的 ref 追加到 tenant-scoped 本地 binding ledger；M3-6 又在隔离 Gravitino Basic IdP 中验证 bounded table-create、catalog-create 拒绝、密码轮换/用户吊销和完整清理；M3-7 已冻结 production identity profile/attestation gate；M3-8 已验证同一 Basic role 与 Iceberg JDBC table 在 PostgreSQL/Gravitino Pod restart 后保持 | harvester 结果、合成 response、本地 sandbox/recovery/metrics/policy/ingestion/identity/JDBC restart observation、projection plan、provider evidence、binding ledger 与 readiness report | 源系统技术对象是原始证据；Gravitino 映射并联邦，不能覆盖业务 ResourceVersion；GDA binding ledger 只记录已验证关系，本地 memory/JDBC catalog evidence 不得冒充生产持久技术权威；Basic IdP、loopback HTTP、file warehouse、pending profile 和合成 attestation 都不是生产身份或生产 storage | Metadata Platform | AR-1 M1/M2 + M3-6 identity + M3-7 gate + M3-8 local persistence 已验证 -> 受保护身份/生产 catalog/attestation/conformance 待执行 | | 治理目录 | M1 已冻结 OpenMetadata table ref/reconciliation；M2 已验证本地 foundation/recovery/metrics/policy 和 production readiness contracts；M3-2 已用 bootstrap admin 创建目标并回读真实 UUID；M3-3 将该 UUID 经 evidence gate 追加到本地 GDA binding ledger；M3-4 将精确 OpenLineage candidate 经 outbox 投递到本地 HTTP receiver；M3-5 已验证临时非管理员 bot 的 scoped `table/Create` grant、policy-create 拒绝及 JWT 轮换/吊销；M3-7 将其 allow/deny 范围纳入双 provider production identity gate | 搜索/页面视图、合成 response、本地 sandbox/recovery/metrics/policy/ingestion/identity observation、projection/provider evidence、binding ledger、lineage outbox/receipt、OpenLineage event 与 readiness report | OpenMetadata 为 owner/glossary/classification/quality discoverability 权威；GDA ledger 保留审批/provider identity，outbox 只拥有投递状态，receiver 拥有接收状态；pending identity profile 和合成 attestation 均不反写 ResourceVersion 或建立生产权威 | Governance | AR-1 M1/M2 + M3-5 local bounded identity + M3-7 readiness contract 已验证 -> protected identity ingestion/生产持久 binding/受保护 production receiver 待执行 | | 血缘 | `gda_control.lineage_event` 已实现 immutable version edge 和幂等 gateway ingest；`agent_asset_lineage` 旧记录仍是可变 asset edge | OpenMetadata lineage graph、UI DAG | 只有 source/target ResourceVersion 与 event checksum 证据完整的旧记录可形成 eligible plan；目录图只作可重建投影 | Data Platform | AR-1 gateway 已验证 -> adapter 待接入 | | Definition | `gda_control.platform_definition_version` 已绑定 definition ResourceVersion、完整逻辑 hash 和原子 gateway registration；3.4.2 adapter 可编译、创建并上线 provider DAG；binding 已以 append-only `execution_plan` Artifact 持久化并可按 tenant + artifact UUID 读取，旧 workflow/template/YAML 仍在写入 | 编辑器状态、DolphinScheduler DAG/definition | 旧 workflow 必须规范化并完整 hash 后才可形成 PlatformDefinitionVersion；provider binding 作为 ExecutionPlanArtifact/evidence，不可反写 definition | DataOps | AR-1 binding persistence 代码已验证 -> staging 调用链待验收 | @@ -34,7 +34,7 @@ | 事件交付 | Standards outbox 已数据库耐久；`gda_control.platform_command_outbox` 已为 DolphinScheduler dispatch/reconcile 提供 tenant RLS、lease claim、幂等 callback、薄 consumer library 和 managed worker process；其他 WebSocket/bot/feedback 多为 best effort | command delivery status、消费者 claim、worker status JSON、WebSocket 消息 | command/event 与源事实同事务入 outbox，幂等 consumer 交付；worker status、outbox 状态、缓存或 socket 都不是 Run/业务权威 | Platform/Integrations | AR-1 command delivery/worker 代码已验证 -> staging worker/callback 待部署 | | 质量结果 | `gda_control.quality_result` 已提供 tenant RLS、append-only gateway 写入，绑定 Run、output ResourceVersion、rule version、verdict、metrics、evidence Artifact 和独立 evaluator；standards、QC、MMFE 专项结果仍未迁移 | dashboard、OpenMetadata quality summary | GDA ledger 保存产品终局所需的不可变 verdict/evidence；OpenMetadata 与 UI 只作可重建发现投影；旧结果缺稳定版本和证据时不得升级为终局依据 | Governance/DataOps | AR-1 最小成功证据已验证 -> 真实规则/staging 待接入 | | 标准与语义定义 | `std_*`、semantic registry 和 YAML 共同存在，生命周期未统一 | prompt/context、搜索索引 | 版本化 Standard/SemanticDefinition 经审批后为权威；Agent context 只消费批准版本 | Governance | AR-1 -> AR-3 | -| 身份与权限 | Chainlit user 可显式绑定 tenant；versioned API 从认证 principal 派生 SubjectContext；`gda_control_gateway` 是 non-login/non-bypass 最小权限角色；Run 可引用强类型 PolicyDecision/Approval Artifact；M3-5/M3-6 分别验证本地 provider scoped grant、越权拒绝和 credential rotation/revocation；M3-7 已冻结生产 OIDC/workload/tenant binding、TLS、持久 catalog 与 attestation contract，但 40 个外部输入仍 blocked | session/cache、前端菜单权限、本地 provider identity evidence、pending profile 与合成 readiness report | IdP/workload identity 提供真实 service identity；PolicyDecision/Approval 继续绑定不可变资源与 execution plan；只有 fresh protected attestation 可派生双 provider production identity claims，profile、Basic/JWT evidence 或人工批准均不可替代 | Security | AR-1 local identities + production readiness contract 已验证 -> protected 双 provider IAM/attestation 待执行 | +| 身份与权限 | Chainlit user 可显式绑定 tenant；versioned API 从认证 principal 派生 SubjectContext；`gda_control_gateway` 是 non-login/non-bypass 最小权限角色；Run 可引用强类型 PolicyDecision/Approval Artifact；M3-5/M3-6 分别验证本地 provider scoped grant、越权拒绝和 credential rotation/revocation；M3-7 已冻结生产 OIDC/workload/tenant binding、TLS、持久 catalog 与 attestation contract，但 40 个外部输入仍 blocked；M3-8 证明同一 Gravitino Basic role 在本地 JDBC restart 后连续 | session/cache、前端菜单权限、本地 provider identity/JDBC restart evidence、pending profile 与合成 readiness report | IdP/workload identity 提供真实 service identity；PolicyDecision/Approval 继续绑定不可变资源与 execution plan；只有 fresh protected attestation 可派生双 provider production identity claims，profile、Basic/JWT evidence、restart continuity 或人工批准均不可替代 | Security | AR-1 local identities/persistence + production readiness contract 已验证 -> protected 双 provider IAM/attestation 待执行 | | GIS 服务定义与 active revision | Martin、REST/MVT/STAC endpoints 和配置直接暴露 | Ingress、tile cache、客户端图层 | GIS Service Control Plane 管 Service/Layer/Style/TMS/DeploymentRevision；provider/Gateway 仅执行 | GIS Platform | AR-4 | | 缓存与进度 | Redis、进程内 dict/task map | UI progress、tile/context cache | 永不作为资产、Run、workflow 或产品权威；丢失后必须从 ledger/provider 重建 | SRE/Platform | 持续约束 | | Agent/Prompt/Model bundle | 多个 registry 与 YAML 存在，尚无统一 deployment revision | trace、eval dashboard | AgentSpecBundle + EvaluationBinding + DeploymentRevision；只消费已发布 DataProductVersion | AgentOps | AR-5 | @@ -56,7 +56,7 @@ 11. `platform_command_outbox` 只拥有投递状态；callback 只触发 reconcile，不能把 provider payload 直接写成 PlatformRun 状态或平台终局。 12. QualityResult evaluator 必须是 workload，且成功终局中的 evaluator 不能等于 Run workload；该代码级职责分离不替代生产 IAM。 13. `candidate_validated`、`registry_subject_bound`、GitHub provenance action 成功、CI artifact、离线 preflight、未独立 attested 的 live observation JSON 或人工批准都不能单独授权 production；缺少同一 source revision 的 OCI subject 独立验证、registry/live revision/identity/health/golden-slice 绑定及受保护 provenance 时，promotion 必须失败。 -14. Metadata Fabric M1 只允许 OpenMetadata/Gravitino GET；M2 只执行本地 foundation/recovery/metrics/policy 演练或验证 production readiness profile；M3-1 只从 synthetic terminal evidence 生成 plan/candidate；M3-2 只允许 exact local PolicyDecision/Approval 后向本地 provider 写 projection；M3-3 只将同一 source evidence 经 PlatformGateway 写入临时 append-only binding ledger；M3-4 只经 tenant-scoped outbox 向无认证 loopback receiver 投递精确 candidate，并验证 at-least-once + receiver idempotency；M3-5 只证明临时 OpenMetadata bot 在 provider 强制 `DefaultBotRole` 之上的项目新增 grant 是 `table/Create`，并验证 policy-create 拒绝与本地 JWT 轮换/吊销；M3-6 只证明隔离 Gravitino Basic user 的 bounded table-create、catalog-create 拒绝、密码轮换/用户吊销和完整清理；M3-7 只冻结 production identity profile、精确 attestation binding 与 fail-closed 派生 claims，既不部署 identity path，也不提交真实 production attestation。Gravitino `1.3.0` Basic IdP 不算 OIDC，生产必须明确选择并证明 custom OIDC authenticator 或 identity-aware proxy。本地 bootstrap provisioner、Basic IdP、loopback HTTP、memory catalog、临时 identity/ledger/outbox、loopback receiver、pending profile、合成 attestation 和 local evidence 都不等于双 provider/生产最小权限、protected workload identity/OIDC、持久 catalog/binding、TLS、受保护 OpenLineage receiver、tenant isolation、alert/SLO、生产 ingestion/conformance 或生产写权威。 +14. Metadata Fabric M1 只允许 OpenMetadata/Gravitino GET；M2 只执行本地 foundation/recovery/metrics/policy 演练或验证 production readiness profile；M3-1 只从 synthetic terminal evidence 生成 plan/candidate；M3-2 只允许 exact local PolicyDecision/Approval 后向本地 provider 写 projection；M3-3 只将同一 source evidence 经 PlatformGateway 写入临时 append-only binding ledger；M3-4 只经 tenant-scoped outbox 向无认证 loopback receiver 投递精确 candidate，并验证 at-least-once + receiver idempotency；M3-5 只证明临时 OpenMetadata bot 在 provider 强制 `DefaultBotRole` 之上的项目新增 grant 是 `table/Create`，并验证 policy-create 拒绝与本地 JWT 轮换/吊销；M3-6 只证明隔离 Gravitino Basic user 的 bounded table-create、catalog-create 拒绝、密码轮换/用户吊销和完整清理；M3-7 只冻结 production identity profile、精确 attestation binding 与 fail-closed 派生 claims，既不部署 identity path，也不提交真实 production attestation；M3-8 只证明 Docker Desktop 单集群中 Basic role、PostgreSQL JDBC metadata 与 file warehouse PVC 在受控 Pod restart 后连续。Gravitino `1.3.0` Basic IdP 不算 OIDC，生产必须明确选择并证明 custom OIDC authenticator 或 identity-aware proxy。本地 bootstrap provisioner、Basic IdP、loopback HTTP、memory/file-backed JDBC catalog、临时 identity/ledger/outbox、loopback receiver、pending profile、合成 attestation 和 local evidence 都不等于双 provider/生产最小权限、protected workload identity/OIDC、生产持久 catalog/binding、TLS、受保护 OpenLineage receiver、tenant isolation、alert/SLO、生产 ingestion/conformance 或生产写权威。 ## 已建立的 AR-0/AR-1 entry 证据 @@ -87,6 +87,7 @@ - Metadata Fabric M3-5 已创建临时 OpenMetadata 非管理员 bot，effective roles 只有 provider 强制 `DefaultBotRole` 与项目 role，项目 policy 只有 `table/Create`；table create/read 为 201/200，policy create 为 403，旧 JWT 轮换后为 401，新 JWT 吊销后为 401，六项 provider cleanup 与 port-forward cleanup 全部通过。evidence SHA 为 `61b6a3429ae948f563bfc2bd012d8b586be581704cec646fd5e74b991243f03f`。`local_openmetadata_minimum_privilege_verified=true` 只描述强制 default role 之上的项目新增 grant；`provider_minimum_privilege_verified=false`、`protected_workload_identity_verified=false`、`oidc_verified=false`、`gravitino_authentication_verified=false`、`production_identity_verified=false`、`production_ready=false`。 - Metadata Fabric M3-6 已在隔离 Gravitino `1.3.0` Basic IdP 中将 bounded user 限定为 `lakehouse` 的 `USE_CATALOG` 与 `lakehouse.published` 的 `USE_SCHEMA`/`CREATE_TABLE`；table create/read 为 200/200，catalog create 为 403，旧密码轮换后为 401，用户删除后替换密码为 401，临时 provider 对象、namespace 和 port-forward 全部清理。evidence SHA 为 `f0b0de1f80f079d43318937e0a0cc151a8546e9e307bef204738b1367f9b29fd`。`local_gravitino_minimum_privilege_verified=true` 只描述本地 Basic rehearsal；OIDC、TLS、持久 catalog 和 production identity 均仍为 `false`。 - Metadata Fabric M3-7 已建立 production identity profile/attestation gate；checked-in profile fingerprint 为 `2e9d5cac3560b853820f923669f6794ead63bcb36a528639fc0e9539e148ee2f`，report fingerprint 为 `c607589ee25a87acc8a1ab71372618a9a4c10c1e8ebff15b8db7e78b37600b9f`，`profile_valid=true`，40 项 federation/provider/TLS/catalog/tenancy/operations 外部输入以 blockers 暴露，`ready_for_protected_verification=false`、`production_identity_gate_passed=false`、`production_ready=false`。合成完整 attestation 只验证门禁逻辑，不计入生产证据。 +- Metadata Fabric M3-8 已在隔离 namespace 将 Iceberg catalog metadata 落到 PostgreSQL JDBC、warehouse 落到独立 PVC；PostgreSQL/Gravitino Pod UID 在顺序 restart 后均变化，StatefulSet/PVC UID 保持，同一 bounded Basic user 重认证并读取相同 table fingerprint，catalog create 前后均为 403，临时 namespace/PV 清理完成。contract fingerprint 为 `f622d8a61bae49171bc76a16bfe64280c616c028bddf88479f1ad04acb1dadf0`，evidence fingerprint 为 `34792bb47ad71041a87adeb644439bf9b6aa3f4855cdc98782d6e3b4282bf1aa`。这不证明 protected workload identity、OIDC/TLS、生产 failure domain、backup/PITR、Spark/Flink conformance、生产 ingestion 或 `production_ready`。 ## 下一验收证据 @@ -95,5 +96,5 @@ - staging 的 migration role、应用 login membership、连接池 role/tenant 复位、双租户 API 和 success finalization 运行产物； - DolphinScheduler adapter 的真实 IAM/OIDC、service token provisioning/轮换、provider 最小权限、binding artifact staging 接入、managed outbox worker/provider callback 实际扩容部署、唯一 worker ID、status/lease 故障恢复和无双写证据； - 首条真实图斑链对 golden slice 的 output hash、独立质量结果/evidence、血缘、发布 revision 和 rollback 演练； -- OpenMetadata/Gravitino 的 source host/cluster 外生产 backup account/bucket、KMS/TLS/workload identity、PITR/source-loss recovery、RPO/RTO、OIDC、受保护环境 provider NetworkPolicy/tenant isolation、upgrade/rollback、registry provenance、持续 metrics backend/retention/query、真实 alert delivery/SLO owner/runbook，以及受保护 PolicyDecision/Approval、双 provider 最小权限 ingestion、生产持久 binding、受保护 production OpenLineage receiver、无双写 read-back 和 conformance；M1 fixture、M2 本地 evidence/readiness contracts、M3-1 projection candidate、M3-2 local replay、M3-3 临时 binding ledger、M3-4 loopback delivery、M3-5/M3-6 本地临时 provider identity 与 M3-7 pending profile/合成 attestation 均不计入生产退出门； +- OpenMetadata/Gravitino 的 source host/cluster 外生产 backup account/bucket、KMS/TLS/workload identity、PITR/source-loss recovery、RPO/RTO、OIDC、受保护环境 provider NetworkPolicy/tenant isolation、upgrade/rollback、registry provenance、持续 metrics backend/retention/query、真实 alert delivery/SLO owner/runbook，以及受保护 PolicyDecision/Approval、双 provider 最小权限 ingestion、生产持久 binding、受保护 production OpenLineage receiver、无双写 read-back 和 conformance；M1 fixture、M2 本地 evidence/readiness contracts、M3-1 projection candidate、M3-2 local replay、M3-3 临时 binding ledger、M3-4 loopback delivery、M3-5/M3-6 本地临时 provider identity、M3-7 pending profile/合成 attestation 与 M3-8 本地 JDBC restart continuity 均不计入生产退出门； - DolphinScheduler/Temporal sandbox 的独立数据库、备份恢复、身份、版本和升级责任证明；DolphinScheduler standalone/H2 不计入此退出门。 diff --git a/k8s/metadata-fabric-gravitino-jdbc-restart/kustomization.yaml b/k8s/metadata-fabric-gravitino-jdbc-restart/kustomization.yaml new file mode 100644 index 00000000..43bd5cbe --- /dev/null +++ b/k8s/metadata-fabric-gravitino-jdbc-restart/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: gda-metadata-catalog-persistence +resources: + - namespace.yaml + - runtime.yaml +labels: + - pairs: + app.kubernetes.io/part-of: gda-metadata-fabric-gravitino-jdbc-restart + gda.openai.com/environment: jdbc-catalog-restart-rehearsal + includeSelectors: false diff --git a/k8s/metadata-fabric-gravitino-jdbc-restart/namespace.yaml b/k8s/metadata-fabric-gravitino-jdbc-restart/namespace.yaml new file mode 100644 index 00000000..f913e68f --- /dev/null +++ b/k8s/metadata-fabric-gravitino-jdbc-restart/namespace.yaml @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: gda-metadata-catalog-persistence + labels: + pod-security.kubernetes.io/enforce: baseline + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/warn: restricted +--- +apiVersion: v1 +kind: ResourceQuota +metadata: + name: gravitino-jdbc-restart-rehearsal + namespace: gda-metadata-catalog-persistence +spec: + hard: + requests.cpu: "3" + requests.memory: 4Gi + limits.cpu: "6" + limits.memory: 8Gi + pods: "6" + persistentvolumeclaims: "4" + requests.storage: 6Gi + secrets: "4" + configmaps: "6" diff --git a/k8s/metadata-fabric-gravitino-jdbc-restart/runtime.yaml b/k8s/metadata-fabric-gravitino-jdbc-restart/runtime.yaml new file mode 100644 index 00000000..5bdf138d --- /dev/null +++ b/k8s/metadata-fabric-gravitino-jdbc-restart/runtime.yaml @@ -0,0 +1,323 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gravitino-persistence-postgresql + namespace: gda-metadata-catalog-persistence +automountServiceAccountToken: false +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gravitino-persistence + namespace: gda-metadata-catalog-persistence +automountServiceAccountToken: false +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: gravitino-persistence-iceberg-init + namespace: gda-metadata-catalog-persistence +data: + 002-iceberg.sql: | + CREATE DATABASE iceberg OWNER gravitino; +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: gravitino-persistence-config + namespace: gda-metadata-catalog-persistence +data: + gravitino.conf: | + gravitino.server.shutdown.timeout = 30000 + gravitino.server.webserver.host = 0.0.0.0 + gravitino.server.webserver.httpPort = 8090 + gravitino.server.health.entityStore.probeTimeoutMs = 5000 + gravitino.entity.store = relational + gravitino.entity.store.relational = JDBCBackend + gravitino.entity.store.relational.jdbcUrl = jdbc:postgresql://gravitino-persistence-postgresql:5432/gravitino + gravitino.entity.store.relational.jdbcDriver = org.postgresql.Driver + gravitino.entity.store.relational.jdbcUser = gravitino + gravitino.authenticators = basic + gravitino.server.rest.extensionPackages = org.apache.gravitino.idp.web.rest.feature + gravitino.authorization.enable = true + gravitino.authorization.serviceAdmins = gda-persistence-admin + gravitino.cache.enabled = false + gravitino.auxService.names = +--- +apiVersion: v1 +kind: Service +metadata: + name: gravitino-persistence-postgresql + namespace: gda-metadata-catalog-persistence +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: gravitino-persistence-postgresql + ports: + - name: postgresql + port: 5432 + targetPort: postgresql +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: gravitino-persistence-postgresql + namespace: gda-metadata-catalog-persistence +spec: + serviceName: gravitino-persistence-postgresql + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: gravitino-persistence-postgresql + template: + metadata: + labels: + app.kubernetes.io/name: gravitino-persistence-postgresql + app.kubernetes.io/component: database + spec: + serviceAccountName: gravitino-persistence-postgresql + automountServiceAccountToken: false + terminationGracePeriodSeconds: 30 + securityContext: + fsGroup: 999 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + containers: + - name: postgresql + image: postgres:16.10-bookworm + imagePullPolicy: IfNotPresent + env: + - name: POSTGRES_USER + value: gravitino + - name: POSTGRES_DB + value: gravitino + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: gravitino-persistence-runtime + key: database-password + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + ports: + - name: postgresql + containerPort: 5432 + startupProbe: + exec: + command: ["pg_isready", "-U", "gravitino", "-d", "gravitino"] + periodSeconds: 3 + failureThreshold: 60 + readinessProbe: + exec: + command: ["pg_isready", "-U", "gravitino", "-d", "gravitino"] + periodSeconds: 5 + failureThreshold: 12 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi + securityContext: + runAsNonRoot: true + runAsUser: 999 + runAsGroup: 999 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: data + mountPath: /var/lib/postgresql/data + - name: runtime + mountPath: /var/run/postgresql + - name: tmp + mountPath: /tmp + - name: schema + mountPath: /docker-entrypoint-initdb.d/001-schema.sql + subPath: 001-schema.sql + readOnly: true + - name: iceberg-init + mountPath: /docker-entrypoint-initdb.d/002-iceberg.sql + subPath: 002-iceberg.sql + readOnly: true + volumes: + - name: runtime + emptyDir: {} + - name: tmp + emptyDir: {} + - name: schema + configMap: + name: gravitino-persistence-schema + - name: iceberg-init + configMap: + name: gravitino-persistence-iceberg-init + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: standard + resources: + requests: + storage: 2Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: gravitino-persistence + namespace: gda-metadata-catalog-persistence +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: gravitino-persistence + ports: + - name: http + port: 8090 + targetPort: http +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: gravitino-persistence + namespace: gda-metadata-catalog-persistence +spec: + serviceName: gravitino-persistence + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: gravitino-persistence + template: + metadata: + labels: + app.kubernetes.io/name: gravitino-persistence + app.kubernetes.io/component: technical-metadata + spec: + serviceAccountName: gravitino-persistence + automountServiceAccountToken: false + terminationGracePeriodSeconds: 60 + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + initContainers: + - name: stage-postgresql-jdbc-driver + image: gda/gravitino:1.3.0-local-arm64 + imagePullPolicy: Never + command: ["/bin/sh", "-ec"] + args: + - install -m 0444 /opt/gravitino/libs/postgresql-42.7.0.jar /driver/postgresql-42.7.0.jar + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 100m + memory: 128Mi + securityContext: + runAsNonRoot: true + runAsUser: 1000 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: jdbc-driver + mountPath: /driver + containers: + - name: gravitino + image: gda/gravitino:1.3.0-local-arm64 + imagePullPolicy: Never + command: ["/bin/bash", "-ec"] + args: + - | + install -d -m 0700 /var/run/gravitino/conf + install -m 0600 /etc/gravitino/gravitino.conf /var/run/gravitino/conf/gravitino.conf + install -m 0444 /opt/gravitino/conf/gravitino-env.sh /var/run/gravitino/conf/gravitino-env.sh + printf '\ngravitino.entity.store.relational.jdbcPassword = ' >> /var/run/gravitino/conf/gravitino.conf + cat /var/run/secrets/gravitino/database-password >> /var/run/gravitino/conf/gravitino.conf + printf '\n' >> /var/run/gravitino/conf/gravitino.conf + exec /opt/gravitino/bin/gravitino.sh --config /var/run/gravitino/conf run + env: + - name: GRAVITINO_MEM + value: -Xms384m -Xmx1024m -XX:MaxMetaspaceSize=384m + - name: GRAVITINO_INITIAL_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: gravitino-persistence-runtime + key: admin-password + ports: + - name: http + containerPort: 8090 + startupProbe: + httpGet: + path: /api/health/ready + port: http + periodSeconds: 5 + failureThreshold: 120 + readinessProbe: + httpGet: + path: /api/health/ready + port: http + periodSeconds: 10 + failureThreshold: 12 + resources: + requests: + cpu: 250m + memory: 768Mi + limits: + cpu: "2" + memory: 2Gi + securityContext: + runAsNonRoot: true + runAsUser: 1000 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: config + mountPath: /etc/gravitino + readOnly: true + - name: secrets + mountPath: /var/run/secrets/gravitino + readOnly: true + - name: runtime + mountPath: /var/run/gravitino + - name: logs + mountPath: /opt/gravitino/logs + - name: tmp + mountPath: /tmp + - name: warehouse + mountPath: /var/lib/gravitino/warehouse + - name: jdbc-driver + mountPath: /opt/gravitino/catalogs/lakehouse-iceberg/libs/postgresql-42.7.0.jar + subPath: postgresql-42.7.0.jar + readOnly: true + volumes: + - name: config + configMap: + name: gravitino-persistence-config + - name: secrets + secret: + secretName: gravitino-persistence-runtime + defaultMode: 0400 + - name: runtime + emptyDir: {} + - name: logs + emptyDir: {} + - name: tmp + emptyDir: {} + - name: jdbc-driver + emptyDir: {} + volumeClaimTemplates: + - metadata: + name: warehouse + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: standard + resources: + requests: + storage: 1Gi diff --git a/scripts/metadata-fabric-gravitino-jdbc-restart.sh b/scripts/metadata-fabric-gravitino-jdbc-restart.sh new file mode 100755 index 00000000..ba3e12ab --- /dev/null +++ b/scripts/metadata-fabric-gravitino-jdbc-restart.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +COMMON_GIT_DIR="$(git -C "$ROOT" rev-parse --path-format=absolute --git-common-dir 2>/dev/null || true)" +SHARED_ROOT="" +if [ -n "$COMMON_GIT_DIR" ]; then + SHARED_ROOT="$(cd "$COMMON_GIT_DIR/.." && pwd)" +fi + +if [ -n "${PYTHON:-}" ]; then + : +elif [ -x "$ROOT/.venv/bin/python" ]; then + PYTHON="$ROOT/.venv/bin/python" +elif [ -n "$SHARED_ROOT" ] && [ -x "$SHARED_ROOT/.venv/bin/python" ]; then + PYTHON="$SHARED_ROOT/.venv/bin/python" +else + PYTHON="python" +fi + +cd "$ROOT" +exec "$PYTHON" -m data_agent.metadata_fabric_gravitino_jdbc_restart "$@"