From d43bdf62068b2308e13b51764e77e0d6e1802cbd Mon Sep 17 00:00:00 2001 From: Ning Zhou Date: Tue, 28 Jul 2026 23:56:37 +0800 Subject: [PATCH] feat(platform): gate metadata provider identity --- .github/workflows/ci.yml | 5 + .../metadata-fabric-identity.production.yaml | 132 +++ data_agent/metadata_fabric_identity_gate.py | 882 ++++++++++++++++++ .../test_metadata_fabric_identity_gate.py | 394 ++++++++ ...metadata-fabric-identity-readiness-gate.md | 93 ++ docs/roadmap-ar0-platform-truth-2026-07-24.md | 5 +- docs/roadmap.md | 2 +- docs/system-of-record-matrix-2026-07-24.md | 16 +- scripts/metadata-fabric-identity-gate.sh | 22 + 9 files changed, 1541 insertions(+), 10 deletions(-) create mode 100644 config/metadata-fabric-identity.production.yaml create mode 100644 data_agent/metadata_fabric_identity_gate.py create mode 100644 data_agent/test_metadata_fabric_identity_gate.py create mode 100644 docs/architecture-decisions/adr-053-production-metadata-fabric-identity-readiness-gate.md create mode 100755 scripts/metadata-fabric-identity-gate.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d87e1dfb..e03fb8e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,7 @@ on: - feat/ar1-metadata-fabric-openlineage-delivery - feat/ar1-metadata-fabric-provider-identity - feat/ar1-metadata-fabric-gravitino-identity + - feat/ar1-metadata-fabric-identity-readiness-gate env: PYTHON_VERSION: "3.13" @@ -137,6 +138,9 @@ jobs: - name: Validate metadata fabric Gravitino identity evidence run: python -m data_agent.metadata_fabric_gravitino_identity validate + - name: Validate metadata fabric production identity gate + run: python -m data_agent.metadata_fabric_identity_gate validate + - name: Validate DolphinScheduler adapter boundary run: python -m data_agent.dolphinscheduler_adapter validate @@ -194,6 +198,7 @@ jobs: data_agent/test_metadata_fabric_lineage_delivery.py \ data_agent/test_metadata_fabric_provider_identity.py \ data_agent/test_metadata_fabric_gravitino_identity.py \ + data_agent/test_metadata_fabric_identity_gate.py \ data_agent/test_metadata_fabric_otel_failure_rehearsal.py \ data_agent/test_metadata_fabric_otel_metrics.py \ data_agent/test_metadata_fabric_provider_metrics.py \ diff --git a/config/metadata-fabric-identity.production.yaml b/config/metadata-fabric-identity.production.yaml new file mode 100644 index 00000000..9177b25b --- /dev/null +++ b/config/metadata-fabric-identity.production.yaml @@ -0,0 +1,132 @@ +schema: gda.metadata_fabric_identity_production_profile.v1 +environment: production + +scope: + providers: + openmetadata: + version: 1.13.1 + gravitino: + version: 1.3.0 + local_evidence: + openmetadata: + path: docs/evidence/metadata-fabric-provider-identity-2026-07-28.json + evidence_fingerprint: 61b6a3429ae948f563bfc2bd012d8b586be581704cec646fd5e74b991243f03f + gravitino: + path: docs/evidence/metadata-fabric-gravitino-identity-2026-07-28.json + evidence_fingerprint: f0b0de1f80f079d43318937e0a0cc151a8546e9e307bef204738b1367f9b29fd + +federation: + decision_status: pending + issuer: null + discovery_uri: null + jwks_uri: null + audience: null + token_exchange_mode: null + trust_policy_reference: null + workload_subject_claim: sub + tenant_claim: gda_tenant_id + maximum_token_ttl_seconds: 900 + +providers: + openmetadata: + integration_mode: null + authentication_component_reference: null + environment_binding: null + workload_identity_reference: null + kubernetes_service_account: null + namespace_template: null + direct_access_policy: deny_except_attested_identity_path + credential_delivery: short_lived_token_exchange + static_credentials_forbidden: true + rotation_mode: automatic_before_expiry + revocation_mode: idp_subject_disable + gravitino: + integration_mode: null + authentication_component_reference: null + environment_binding: null + workload_identity_reference: null + kubernetes_service_account: null + namespace_template: null + direct_access_policy: deny_except_attested_identity_path + credential_delivery: short_lived_token_exchange + static_credentials_forbidden: true + rotation_mode: automatic_before_expiry + revocation_mode: idp_subject_disable + +authorization: + openmetadata: + mandatory_roles: + - DefaultBotRole + project_role: GdaMetadataTableProjectionRole + project_permissions: + - resource: table + operations: + - Create + denied_probe: + resource: policy + operation: Create + gravitino: + role: gda-table-projection + securable_objects: + - full_name: lakehouse + type: CATALOG + privileges: + - USE_CATALOG + - full_name: lakehouse.published + type: SCHEMA + privileges: + - CREATE_TABLE + - USE_SCHEMA + denied_probe: + resource: METALAKE + operation: CREATE_CATALOG + +tls: + required: true + minimum_version: TLSv1.2 + openmetadata_endpoint: null + gravitino_endpoint: null + trust_bundle_reference: null + certificate_policy_reference: null + mtls_for_internal_hops: true + +catalog: + decision_status: pending + gravitino_backend: null + catalog_reference: null + persistence_reference: null + backup_policy_reference: null + persistent_required: true + +tenancy: + isolation_mode: null + policy_reference: null + tenant_claim: gda_tenant_id + cross_tenant_denial_required: true + +operations: + identity_owner: null + security_owner: null + incident_owner: null + audit_log_reference: null + rotation_slo_minutes: null + revocation_slo_minutes: null + runbook: + uri: null + version: null + rollback_runbook: + uri: null + version: null + +claims: + identity_decision_frozen: false + provider_minimum_privilege_verified: false + protected_workload_identity_verified: false + oidc_verified: false + tls_verified: false + credential_rotation_verified: false + credential_revocation_verified: false + persistent_catalog_identity_binding_verified: false + production_identity_verified: false + production_identity_gate_passed: false + production_ready: false diff --git a/data_agent/metadata_fabric_identity_gate.py b/data_agent/metadata_fabric_identity_gate.py new file mode 100644 index 00000000..3e60673c --- /dev/null +++ b/data_agent/metadata_fabric_identity_gate.py @@ -0,0 +1,882 @@ +"""Evaluate the fail-closed production Metadata Fabric identity gate. + +The checked-in profile freezes the decisions and protected-environment checks +required to promote the local OpenMetadata and Gravitino identity rehearsals. +Missing decisions are explicit blockers. A production identity result requires +a fresh attestation bound to the exact profile and all derived bindings. This +module deploys nothing, accepts no credentials, and never claims that the whole +platform is production ready. +""" + +from __future__ import annotations + +import argparse +import ipaddress +import json +import re +import sys +from collections.abc import Mapping +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +import yaml + +from . import metadata_fabric_recovery_rehearsal as recovery + + +PROFILE_SCHEMA = "gda.metadata_fabric_identity_production_profile.v1" +ATTESTATION_SCHEMA = "gda.metadata_fabric_identity_attestation.v1" +REPORT_SCHEMA = "gda.metadata_fabric_identity_readiness_report.v1" +ENVIRONMENT = "production" + +REPO_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_PROFILE_PATH = REPO_ROOT / "config/metadata-fabric-identity.production.yaml" + +EXPECTED_PROVIDERS = {"openmetadata": "1.13.1", "gravitino": "1.3.0"} +EXPECTED_LOCAL_EVIDENCE = { + "openmetadata": { + "path": "docs/evidence/metadata-fabric-provider-identity-2026-07-28.json", + "evidence_fingerprint": ( + "61b6a3429ae948f563bfc2bd012d8b586be581704cec646fd5e74b991243f03f" + ), + }, + "gravitino": { + "path": "docs/evidence/metadata-fabric-gravitino-identity-2026-07-28.json", + "evidence_fingerprint": ( + "f0b0de1f80f079d43318937e0a0cc151a8546e9e307bef204738b1367f9b29fd" + ), + }, +} +EXPECTED_AUTHORIZATION = { + "openmetadata": { + "mandatory_roles": ["DefaultBotRole"], + "project_role": "GdaMetadataTableProjectionRole", + "project_permissions": [{"resource": "table", "operations": ["Create"]}], + "denied_probe": {"resource": "policy", "operation": "Create"}, + }, + "gravitino": { + "role": "gda-table-projection", + "securable_objects": [ + { + "full_name": "lakehouse", + "type": "CATALOG", + "privileges": ["USE_CATALOG"], + }, + { + "full_name": "lakehouse.published", + "type": "SCHEMA", + "privileges": ["CREATE_TABLE", "USE_SCHEMA"], + }, + ], + "denied_probe": {"resource": "METALAKE", "operation": "CREATE_CATALOG"}, + }, +} +ALLOWED_INTEGRATION_MODES = { + "openmetadata": {"provider_native_oidc", "identity_aware_proxy"}, + "gravitino": {"custom_oidc_authenticator", "identity_aware_proxy"}, +} +ALLOWED_CATALOG_BACKENDS = {"iceberg_rest", "jdbc"} +EXPECTED_ATTESTATION_CHECKS = { + "oidc_discovery", + "token_exchange", + "workload_subject_binding", + "tenant_claim_binding", + "openmetadata_authenticated_allow", + "openmetadata_administrative_deny", + "gravitino_authenticated_allow", + "gravitino_administrative_deny", + "provider_direct_access_denied", + "static_credential_absence", + "credential_rotation", + "credential_revocation", + "tls_transport", + "mtls_internal_hops", + "persistent_catalog_restart", + "cross_tenant_denial", + "audit_log_delivery", + "rollback_rehearsal", +} +EXPECTED_CLAIMS = { + "identity_decision_frozen", + "provider_minimum_privilege_verified", + "protected_workload_identity_verified", + "oidc_verified", + "tls_verified", + "credential_rotation_verified", + "credential_revocation_verified", + "persistent_catalog_identity_binding_verified", + "production_identity_verified", + "production_identity_gate_passed", + "production_ready", +} +REPORT_CLAIMS = { + "provider_minimum_privilege_verified", + "protected_workload_identity_verified", + "oidc_verified", + "tls_verified", + "credential_rotation_verified", + "credential_revocation_verified", + "persistent_catalog_identity_binding_verified", + "production_identity_verified", + "production_identity_gate_passed", +} +REPORT_INVENTORY = { + "schema", + "environment", + "profile_fingerprint", + "attestation_fingerprint", + "federation_fingerprint", + "provider_bindings_fingerprint", + "authorization_fingerprint", + "tls_fingerprint", + "catalog_fingerprint", + "tenancy_fingerprint", + "profile_valid", + "profile_errors", + "profile_blockers", + "ready_for_protected_verification", + "attestation_valid", + "attestation_errors", + *REPORT_CLAIMS, + "production_ready", + "report_fingerprint", +} + +SHA40_PATTERN = re.compile(r"^[0-9a-f]{40}$") +DNS_LABEL_PATTERN = re.compile(r"^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$") +REFERENCE_PATTERN = re.compile(r"^[a-z][a-z0-9+.-]*://[^\s]+$", re.IGNORECASE) +PINNED_OCI_PATTERN = re.compile(r"^oci://[^\s@]+@sha256:[0-9a-f]{64}$") +PLACEHOLDER_PATTERN = re.compile( + r"(^|[-_.:/])(pending|placeholder|replace|tbd|todo|changeme)([-_.:/]|$)|" + r"[<>]|\.example(?=[:/]|$)", + re.IGNORECASE, +) +SENSITIVE_KEY_PATTERN = re.compile( + r"(^|[-_.])(password|passwd|secret|client[-_.]?secret|private[-_.]?key|" + r"access[-_.]?key|access[-_.]?token|refresh[-_.]?token|id[-_.]?token|" + r"authorization[-_.]?header)($|[-_.])", + re.IGNORECASE, +) + + +class MetadataFabricIdentityGateError(RuntimeError): + """The production identity readiness contract failed closed.""" + + +def _mapping(value: Any) -> Mapping[str, Any]: + return value if isinstance(value, Mapping) else {} + + +def _load_yaml_object(path: Path) -> dict[str, Any]: + value = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError("YAML document is not an object") + return value + + +def _load_json_object(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError("JSON document is not an object") + return value + + +def _inventory_errors( + value: Mapping[str, Any], expected: set[str], label: str +) -> list[str]: + return [] if set(value) == expected else [f"{label} inventory does not match"] + + +def _placeholder(value: Any) -> bool: + return not isinstance(value, str) or not value.strip() or bool( + PLACEHOLDER_PATTERN.search(value.strip()) + ) + + +def _reference(value: Any) -> bool: + return not _placeholder(value) and bool(REFERENCE_PATTERN.fullmatch(str(value))) + + +def _production_https_url(value: Any) -> bool: + if _placeholder(value): + return False + parsed = urlparse(str(value)) + if ( + parsed.scheme != "https" + or not parsed.hostname + or parsed.username + or parsed.password + or parsed.query + or parsed.fragment + ): + return False + hostname = parsed.hostname.lower() + if hostname in {"localhost", "0.0.0.0", "::1"} or hostname.endswith( + (".localhost", ".local", ".svc", ".cluster.local") + ): + return False + if hostname in {"example.com", "example.net", "example.org"} or hostname.endswith( + (".example.com", ".example.net", ".example.org") + ): + return False + try: + if ipaddress.ip_address(hostname).is_private: + return False + except ValueError: + pass + return True + + +def _sensitive_paths(value: Any, path: tuple[str, ...] = ()) -> list[str]: + found: list[str] = [] + if isinstance(value, Mapping): + for key, nested in value.items(): + key_text = str(key) + child = (*path, key_text) + if SENSITIVE_KEY_PATTERN.search(key_text): + found.append(".".join(child)) + found.extend(_sensitive_paths(nested, child)) + elif isinstance(value, list): + for index, nested in enumerate(value): + found.extend(_sensitive_paths(nested, (*path, str(index)))) + return found + + +def _local_evidence_errors(value: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + errors.extend( + _inventory_errors(value, set(EXPECTED_LOCAL_EVIDENCE), "local evidence") + ) + for provider, expected in EXPECTED_LOCAL_EVIDENCE.items(): + item = _mapping(value.get(provider)) + errors.extend( + _inventory_errors( + item, {"path", "evidence_fingerprint"}, f"{provider} local evidence" + ) + ) + if dict(item) != expected: + errors.append(f"{provider} local identity evidence binding does not match") + continue + path = (REPO_ROOT / expected["path"]).resolve() + try: + path.relative_to(REPO_ROOT) + evidence = _load_json_object(path) + except (OSError, TypeError, ValueError, json.JSONDecodeError): + errors.append(f"{provider} local identity evidence is unavailable") + continue + if evidence.get("evidence_fingerprint") != expected["evidence_fingerprint"]: + errors.append(f"{provider} local identity evidence fingerprint drifted") + if evidence.get("production_identity_verified") is not False: + errors.append(f"{provider} local evidence overclaims production identity") + if provider == "openmetadata": + if ( + evidence.get("local_openmetadata_bounded_identity_verified") is not True + or evidence.get("local_openmetadata_minimum_privilege_verified") is not True + ): + errors.append("OpenMetadata local identity evidence is not verified") + elif ( + evidence.get("local_gravitino_basic_identity_verified") is not True + or evidence.get("local_gravitino_minimum_privilege_verified") is not True + ): + errors.append("Gravitino local identity evidence is not verified") + return errors + + +def _profile_errors(profile: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + if _sensitive_paths(profile): + errors.append("identity profile contains credential-bearing fields") + errors.extend( + _inventory_errors( + profile, + { + "schema", + "environment", + "scope", + "federation", + "providers", + "authorization", + "tls", + "catalog", + "tenancy", + "operations", + "claims", + }, + "identity profile", + ) + ) + if profile.get("schema") != PROFILE_SCHEMA or profile.get("environment") != ENVIRONMENT: + errors.append("identity profile schema or environment does not match") + + scope = _mapping(profile.get("scope")) + errors.extend( + _inventory_errors(scope, {"providers", "local_evidence"}, "identity scope") + ) + providers = _mapping(scope.get("providers")) + if set(providers) != set(EXPECTED_PROVIDERS): + errors.append("identity provider version inventory does not match") + for provider, version in EXPECTED_PROVIDERS.items(): + if _mapping(providers.get(provider)) != {"version": version}: + errors.append(f"{provider} version binding does not match") + errors.extend(_local_evidence_errors(_mapping(scope.get("local_evidence")))) + + federation = _mapping(profile.get("federation")) + federation_keys = { + "decision_status", + "issuer", + "discovery_uri", + "jwks_uri", + "audience", + "token_exchange_mode", + "trust_policy_reference", + "workload_subject_claim", + "tenant_claim", + "maximum_token_ttl_seconds", + } + errors.extend(_inventory_errors(federation, federation_keys, "identity federation")) + if federation.get("decision_status") not in {"pending", "approved"}: + errors.append("identity federation decision status is invalid") + for key in ("issuer", "discovery_uri", "jwks_uri"): + value = federation.get(key) + if value is not None and not _production_https_url(value): + errors.append(f"identity federation {key} is not a production HTTPS URL") + for key in ("audience", "trust_policy_reference"): + value = federation.get(key) + if value is not None and not _reference(value): + errors.append(f"identity federation {key} is invalid") + if federation.get("token_exchange_mode") not in { + None, + "oidc_workload_federation", + }: + errors.append("identity token exchange mode is invalid") + if federation.get("workload_subject_claim") != "sub": + errors.append("identity workload subject claim must remain sub") + if federation.get("tenant_claim") != "gda_tenant_id": + errors.append("identity tenant claim must remain gda_tenant_id") + maximum_ttl = federation.get("maximum_token_ttl_seconds") + if ( + not isinstance(maximum_ttl, int) + or isinstance(maximum_ttl, bool) + or not 300 <= maximum_ttl <= 900 + ): + errors.append("identity maximum token TTL must be between 300 and 900 seconds") + + provider_bindings = _mapping(profile.get("providers")) + errors.extend( + _inventory_errors(provider_bindings, set(EXPECTED_PROVIDERS), "provider binding") + ) + provider_keys = { + "integration_mode", + "authentication_component_reference", + "environment_binding", + "workload_identity_reference", + "kubernetes_service_account", + "namespace_template", + "direct_access_policy", + "credential_delivery", + "static_credentials_forbidden", + "rotation_mode", + "revocation_mode", + } + for provider in EXPECTED_PROVIDERS: + item = _mapping(provider_bindings.get(provider)) + errors.extend(_inventory_errors(item, provider_keys, f"{provider} identity binding")) + mode = item.get("integration_mode") + if mode is not None and mode not in ALLOWED_INTEGRATION_MODES[provider]: + errors.append(f"{provider} identity integration mode is invalid") + component = item.get("authentication_component_reference") + if component is not None and not PINNED_OCI_PATTERN.fullmatch(str(component)): + errors.append(f"{provider} authentication component must be digest-pinned OCI") + for key in ("environment_binding", "workload_identity_reference"): + value = item.get(key) + if value is not None and not _reference(value): + errors.append(f"{provider} {key} is invalid") + service_account = item.get("kubernetes_service_account") + if service_account is not None and not DNS_LABEL_PATTERN.fullmatch( + str(service_account) + ): + errors.append(f"{provider} Kubernetes ServiceAccount is invalid") + namespace = item.get("namespace_template") + if namespace is not None and ( + _placeholder(namespace) or "{tenant_id}" not in str(namespace) + ): + errors.append(f"{provider} namespace template must bind tenant_id") + expected_static = { + "direct_access_policy": "deny_except_attested_identity_path", + "credential_delivery": "short_lived_token_exchange", + "static_credentials_forbidden": True, + "rotation_mode": "automatic_before_expiry", + "revocation_mode": "idp_subject_disable", + } + for key, expected in expected_static.items(): + if item.get(key) != expected: + errors.append(f"{provider} identity control does not match: {key}") + + if _mapping(profile.get("authorization")) != EXPECTED_AUTHORIZATION: + errors.append("provider minimum-privilege authorization contract does not match") + + tls = _mapping(profile.get("tls")) + tls_keys = { + "required", + "minimum_version", + "openmetadata_endpoint", + "gravitino_endpoint", + "trust_bundle_reference", + "certificate_policy_reference", + "mtls_for_internal_hops", + } + errors.extend(_inventory_errors(tls, tls_keys, "identity TLS")) + if ( + tls.get("required") is not True + or tls.get("minimum_version") not in {"TLSv1.2", "TLSv1.3"} + or tls.get("mtls_for_internal_hops") is not True + ): + errors.append("identity TLS baseline does not match") + for key in ("openmetadata_endpoint", "gravitino_endpoint"): + value = tls.get(key) + if value is not None and not _production_https_url(value): + errors.append(f"identity TLS {key} is invalid") + for key in ("trust_bundle_reference", "certificate_policy_reference"): + value = tls.get(key) + if value is not None and not _reference(value): + errors.append(f"identity TLS {key} is invalid") + if ( + tls.get("openmetadata_endpoint") is not None + and tls.get("openmetadata_endpoint") == tls.get("gravitino_endpoint") + ): + errors.append("metadata providers must have distinct TLS endpoints") + + catalog = _mapping(profile.get("catalog")) + catalog_keys = { + "decision_status", + "gravitino_backend", + "catalog_reference", + "persistence_reference", + "backup_policy_reference", + "persistent_required", + } + errors.extend(_inventory_errors(catalog, catalog_keys, "identity catalog")) + if catalog.get("decision_status") not in {"pending", "approved"}: + errors.append("identity catalog decision status is invalid") + if catalog.get("gravitino_backend") not in {None, *ALLOWED_CATALOG_BACKENDS}: + errors.append("production Gravitino catalog backend is invalid") + for key in ("catalog_reference", "persistence_reference", "backup_policy_reference"): + value = catalog.get(key) + if value is not None and not _reference(value): + errors.append(f"production catalog {key} is invalid") + if catalog.get("persistent_required") is not True: + errors.append("production Gravitino catalog must require persistence") + + tenancy = _mapping(profile.get("tenancy")) + tenancy_keys = { + "isolation_mode", + "policy_reference", + "tenant_claim", + "cross_tenant_denial_required", + } + errors.extend(_inventory_errors(tenancy, tenancy_keys, "identity tenancy")) + if tenancy.get("isolation_mode") not in {None, "namespace_and_provider_policy"}: + errors.append("identity tenant isolation mode is invalid") + if tenancy.get("policy_reference") is not None and not _reference( + tenancy.get("policy_reference") + ): + errors.append("identity tenant policy reference is invalid") + if ( + tenancy.get("tenant_claim") != "gda_tenant_id" + or tenancy.get("cross_tenant_denial_required") is not True + ): + errors.append("identity tenant contract does not match") + + operations = _mapping(profile.get("operations")) + operation_keys = { + "identity_owner", + "security_owner", + "incident_owner", + "audit_log_reference", + "rotation_slo_minutes", + "revocation_slo_minutes", + "runbook", + "rollback_runbook", + } + errors.extend(_inventory_errors(operations, operation_keys, "identity operations")) + for key in ("identity_owner", "security_owner", "incident_owner", "audit_log_reference"): + value = operations.get(key) + if value is not None and not _reference(value): + errors.append(f"identity operations {key} is invalid") + for key, maximum in (("rotation_slo_minutes", 60), ("revocation_slo_minutes", 15)): + value = operations.get(key) + if value is not None and ( + not isinstance(value, int) + or isinstance(value, bool) + or not 1 <= value <= maximum + ): + errors.append(f"identity operations {key} is invalid") + for name in ("runbook", "rollback_runbook"): + runbook = _mapping(operations.get(name)) + errors.extend(_inventory_errors(runbook, {"uri", "version"}, name)) + if runbook.get("uri") is not None and not _production_https_url( + runbook.get("uri") + ): + errors.append(f"identity {name} URI is invalid") + if runbook.get("version") is not None and _placeholder(runbook.get("version")): + errors.append(f"identity {name} version is invalid") + + claims = _mapping(profile.get("claims")) + errors.extend(_inventory_errors(claims, EXPECTED_CLAIMS, "identity claim")) + for claim in sorted(EXPECTED_CLAIMS): + if claims.get(claim) is not False: + errors.append(f"identity profile may not self-assert production claim: {claim}") + return errors + + +def _profile_blockers(profile: Mapping[str, Any]) -> list[str]: + blockers: list[str] = [] + federation = _mapping(profile.get("federation")) + if federation.get("decision_status") != "approved": + blockers.append("federation.decision_status") + for key in ( + "issuer", + "discovery_uri", + "jwks_uri", + "audience", + "token_exchange_mode", + "trust_policy_reference", + ): + if federation.get(key) is None: + blockers.append(f"federation.{key}") + + providers = _mapping(profile.get("providers")) + for provider in EXPECTED_PROVIDERS: + item = _mapping(providers.get(provider)) + for key in ( + "integration_mode", + "authentication_component_reference", + "environment_binding", + "workload_identity_reference", + "kubernetes_service_account", + "namespace_template", + ): + if item.get(key) is None: + blockers.append(f"providers.{provider}.{key}") + + tls = _mapping(profile.get("tls")) + for key in ( + "openmetadata_endpoint", + "gravitino_endpoint", + "trust_bundle_reference", + "certificate_policy_reference", + ): + if tls.get(key) is None: + blockers.append(f"tls.{key}") + + catalog = _mapping(profile.get("catalog")) + if catalog.get("decision_status") != "approved": + blockers.append("catalog.decision_status") + for key in ( + "gravitino_backend", + "catalog_reference", + "persistence_reference", + "backup_policy_reference", + ): + if catalog.get(key) is None: + blockers.append(f"catalog.{key}") + + tenancy = _mapping(profile.get("tenancy")) + for key in ("isolation_mode", "policy_reference"): + if tenancy.get(key) is None: + blockers.append(f"tenancy.{key}") + + operations = _mapping(profile.get("operations")) + for key in ( + "identity_owner", + "security_owner", + "incident_owner", + "audit_log_reference", + "rotation_slo_minutes", + "revocation_slo_minutes", + ): + if operations.get(key) is None: + blockers.append(f"operations.{key}") + for name in ("runbook", "rollback_runbook"): + runbook = _mapping(operations.get(name)) + for key in ("uri", "version"): + if runbook.get(key) is None: + blockers.append(f"operations.{name}.{key}") + return blockers + + +def _binding_fingerprints(profile: Mapping[str, Any]) -> dict[str, str]: + return { + "federation_fingerprint": recovery._canonical_sha256( + _mapping(profile.get("federation")) + ), + "provider_bindings_fingerprint": recovery._canonical_sha256( + _mapping(profile.get("providers")) + ), + "authorization_fingerprint": recovery._canonical_sha256( + _mapping(profile.get("authorization")) + ), + "tls_fingerprint": recovery._canonical_sha256(_mapping(profile.get("tls"))), + "catalog_fingerprint": recovery._canonical_sha256( + _mapping(profile.get("catalog")) + ), + "tenancy_fingerprint": recovery._canonical_sha256( + _mapping(profile.get("tenancy")) + ), + } + + +def _attestation_errors( + attestation: Mapping[str, Any] | None, + *, + profile: Mapping[str, Any], + profile_fingerprint: str, + bindings: Mapping[str, str], + now: datetime, + max_age: timedelta, +) -> list[str]: + if attestation is None: + return ["production identity attestation is missing"] + errors: list[str] = [] + if _sensitive_paths(attestation): + errors.append("identity attestation contains credential-bearing fields") + expected_inventory = { + "schema", + "environment", + "profile_fingerprint", + "source_revision", + "observed_at", + "expires_at", + "protected_environment", + "verifier_identity", + "evidence_uri", + "provider_versions", + "local_evidence_fingerprints", + *bindings.keys(), + "runbook_versions", + "checks", + } + errors.extend( + _inventory_errors(attestation, expected_inventory, "identity attestation") + ) + if ( + attestation.get("schema") != ATTESTATION_SCHEMA + or attestation.get("environment") != ENVIRONMENT + ): + errors.append("identity attestation schema or environment does not match") + if attestation.get("profile_fingerprint") != profile_fingerprint: + errors.append("identity attestation is not bound to the current profile") + if not SHA40_PATTERN.fullmatch(str(attestation.get("source_revision") or "")): + errors.append("identity attestation source revision is invalid") + if attestation.get("protected_environment") != ENVIRONMENT: + errors.append("identity attestation did not run in protected production") + if not _reference(attestation.get("verifier_identity")): + errors.append("identity attestation verifier identity is invalid") + if not _production_https_url(attestation.get("evidence_uri")): + errors.append("identity attestation evidence URI is invalid") + + try: + observed_at = datetime.fromisoformat(str(attestation.get("observed_at"))) + expires_at = datetime.fromisoformat(str(attestation.get("expires_at"))) + if observed_at.tzinfo is None or observed_at.utcoffset() is None: + raise ValueError + if expires_at.tzinfo is None or expires_at.utcoffset() is None: + raise ValueError + age = now - observed_at + if age < timedelta(seconds=-30) or age > max_age: + errors.append("identity attestation is outside the accepted freshness window") + if expires_at <= now or expires_at <= observed_at: + errors.append("identity attestation has expired or has invalid expiry") + if expires_at - observed_at > timedelta(days=7): + errors.append("identity attestation validity exceeds seven days") + except ValueError: + errors.append("identity attestation timestamps are invalid") + + if _mapping(attestation.get("provider_versions")) != EXPECTED_PROVIDERS: + errors.append("identity attestation provider versions do not match") + local_fingerprints = { + provider: item["evidence_fingerprint"] + for provider, item in EXPECTED_LOCAL_EVIDENCE.items() + } + if _mapping(attestation.get("local_evidence_fingerprints")) != local_fingerprints: + errors.append("identity attestation local evidence bindings do not match") + for key, fingerprint in bindings.items(): + if attestation.get(key) != fingerprint: + errors.append(f"identity attestation binding does not match: {key}") + operations = _mapping(profile.get("operations")) + expected_runbooks = { + name: _mapping(operations.get(name)).get("version") + for name in ("runbook", "rollback_runbook") + } + if _mapping(attestation.get("runbook_versions")) != expected_runbooks: + errors.append("identity attestation runbook versions do not match") + checks = _mapping(attestation.get("checks")) + if set(checks) != EXPECTED_ATTESTATION_CHECKS: + errors.append("identity attestation check inventory does not match") + for check in sorted(EXPECTED_ATTESTATION_CHECKS): + if checks.get(check) != "passed": + errors.append(f"identity attestation check did not pass: {check}") + return errors + + +def _invalid_report(error: str) -> dict[str, Any]: + stable: dict[str, Any] = { + "schema": REPORT_SCHEMA, + "environment": ENVIRONMENT, + "profile_fingerprint": None, + "attestation_fingerprint": None, + "federation_fingerprint": None, + "provider_bindings_fingerprint": None, + "authorization_fingerprint": None, + "tls_fingerprint": None, + "catalog_fingerprint": None, + "tenancy_fingerprint": None, + "profile_valid": False, + "profile_errors": [error], + "profile_blockers": [], + "ready_for_protected_verification": False, + "attestation_valid": False, + "attestation_errors": ["production identity attestation is missing"], + **{claim: False for claim in REPORT_CLAIMS}, + "production_ready": False, + } + return {**stable, "report_fingerprint": recovery._canonical_sha256(stable)} + + +def build_identity_readiness_report( + *, + profile_path: Path | None = None, + attestation: Mapping[str, Any] | None = None, + now: datetime | None = None, + max_attestation_age: timedelta = timedelta(hours=24), +) -> dict[str, Any]: + """Build a deterministic identity readiness report.""" + current = now or datetime.now(UTC) + if current.tzinfo is None or current.utcoffset() is None: + raise MetadataFabricIdentityGateError( + "identity readiness verification time must be timezone-aware" + ) + if max_attestation_age <= timedelta(0): + raise MetadataFabricIdentityGateError( + "identity attestation freshness window must be positive" + ) + path = (profile_path or DEFAULT_PROFILE_PATH).resolve() + try: + profile = _load_yaml_object(path) + except (OSError, TypeError, yaml.YAMLError) as exc: + return _invalid_report(f"identity profile is invalid: {type(exc).__name__}") + + profile_fingerprint = recovery._canonical_sha256(profile) + bindings = _binding_fingerprints(profile) + profile_errors = _profile_errors(profile) + profile_blockers = _profile_blockers(profile) + profile_valid = not profile_errors + ready_for_verification = profile_valid and not profile_blockers + attestation_errors = _attestation_errors( + attestation, + profile=profile, + profile_fingerprint=profile_fingerprint, + bindings=bindings, + now=current, + max_age=max_attestation_age, + ) + attestation_valid = ready_for_verification and not attestation_errors + gate_passed = ready_for_verification and attestation_valid + stable = { + "schema": REPORT_SCHEMA, + "environment": ENVIRONMENT, + "profile_fingerprint": profile_fingerprint, + "attestation_fingerprint": ( + recovery._canonical_sha256(attestation) if attestation is not None else None + ), + **bindings, + "profile_valid": profile_valid, + "profile_errors": profile_errors, + "profile_blockers": profile_blockers, + "ready_for_protected_verification": ready_for_verification, + "attestation_valid": attestation_valid, + "attestation_errors": attestation_errors, + **{claim: gate_passed for claim in REPORT_CLAIMS}, + "production_ready": False, + } + return {**stable, "report_fingerprint": recovery._canonical_sha256(stable)} + + +def verify_report_integrity(report: Mapping[str, Any]) -> list[str]: + """Reject modified identity reports and overall production overclaims.""" + errors: list[str] = [] + if _sensitive_paths(report): + errors.append("identity readiness report contains credential-bearing fields") + errors.extend(_inventory_errors(report, REPORT_INVENTORY, "identity readiness report")) + if report.get("schema") != REPORT_SCHEMA or report.get("environment") != ENVIRONMENT: + errors.append("identity readiness report schema or environment does not match") + stable = {key: value for key, value in report.items() if key != "report_fingerprint"} + if report.get("report_fingerprint") != recovery._canonical_sha256(stable): + errors.append("identity readiness report fingerprint does not match") + if report.get("production_ready") is not False: + errors.append("identity gate may not claim overall production readiness") + expected_gate = ( + report.get("profile_valid") is True + and report.get("profile_blockers") == [] + and report.get("attestation_valid") is True + ) + for claim in sorted(REPORT_CLAIMS): + if report.get(claim) is not expected_gate: + errors.append(f"identity gate result is inconsistent: {claim}") + return errors + + +def _write_report(report: Mapping[str, Any], output: Path | None) -> None: + rendered = json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + if output is None: + print(rendered, end="") + return + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(rendered, encoding="utf-8") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + validate = subparsers.add_parser("validate") + validate.add_argument("--profile", type=Path, default=DEFAULT_PROFILE_PATH) + evaluate = subparsers.add_parser("evaluate") + evaluate.add_argument("--profile", type=Path, default=DEFAULT_PROFILE_PATH) + evaluate.add_argument("--attestation", type=Path, required=True) + evaluate.add_argument("--output", type=Path) + verify = subparsers.add_parser("verify") + verify.add_argument("--input", type=Path, required=True) + args = parser.parse_args(argv) + try: + if args.command == "validate": + report = build_identity_readiness_report(profile_path=args.profile) + _write_report(report, None) + return 0 if report["profile_valid"] else 1 + if args.command == "evaluate": + attestation = _load_json_object(args.attestation) + report = build_identity_readiness_report( + profile_path=args.profile, + attestation=attestation, + ) + _write_report(report, args.output) + return 0 if report["production_identity_gate_passed"] else 1 + report = _load_json_object(args.input) + errors = verify_report_integrity(report) + _write_report({"verified": not errors, "errors": errors}, None) + return 0 if not errors else 1 + except ( + OSError, + TypeError, + ValueError, + json.JSONDecodeError, + yaml.YAMLError, + MetadataFabricIdentityGateError, + ) as exc: + print(f"metadata identity gate: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/data_agent/test_metadata_fabric_identity_gate.py b/data_agent/test_metadata_fabric_identity_gate.py new file mode 100644 index 00000000..f726059b --- /dev/null +++ b/data_agent/test_metadata_fabric_identity_gate.py @@ -0,0 +1,394 @@ +import json +from copy import deepcopy +from datetime import UTC, datetime, timedelta +from pathlib import Path + +import pytest +import yaml + +from data_agent import metadata_fabric_identity_gate as gate + + +NOW = datetime(2026, 7, 28, 16, 0, tzinfo=UTC) + + +def _default_profile() -> dict: + return yaml.safe_load(gate.DEFAULT_PROFILE_PATH.read_text(encoding="utf-8")) + + +def _write_profile(tmp_path: Path, profile: dict) -> Path: + target = tmp_path / "identity-profile.yaml" + target.write_text(yaml.safe_dump(profile, sort_keys=False), encoding="utf-8") + return target + + +def _complete_profile() -> dict: + profile = _default_profile() + profile["federation"].update( + { + "decision_status": "approved", + "issuer": "https://identity.gda.internal", + "discovery_uri": ( + "https://identity.gda.internal/.well-known/openid-configuration" + ), + "jwks_uri": "https://identity.gda.internal/oauth2/jwks", + "audience": "audience://metadata-fabric/production", + "token_exchange_mode": "oidc_workload_federation", + "trust_policy_reference": "policy://production/metadata-federation-v1", + } + ) + for provider, mode in ( + ("openmetadata", "provider_native_oidc"), + ("gravitino", "custom_oidc_authenticator"), + ): + item = profile["providers"][provider] + item.update( + { + "integration_mode": mode, + "environment_binding": f"environment://production/{provider}", + "workload_identity_reference": ( + f"identity://production/metadata/{provider}" + ), + "kubernetes_service_account": provider, + "namespace_template": "gda-{tenant_id}", + } + ) + digest_character = "a" if provider == "openmetadata" else "b" + item["authentication_component_reference"] = ( + f"oci://registry.gda.internal/metadata/{provider}-identity@sha256:" + f"{digest_character * 64}" + ) + profile["tls"].update( + { + "minimum_version": "TLSv1.3", + "openmetadata_endpoint": "https://openmetadata.gda.internal/api", + "gravitino_endpoint": "https://gravitino.gda.internal/api", + "trust_bundle_reference": "pki://production/metadata-fabric-ca-v1", + "certificate_policy_reference": ( + "policy://production/metadata-certificate-v1" + ), + } + ) + profile["catalog"].update( + { + "decision_status": "approved", + "gravitino_backend": "iceberg_rest", + "catalog_reference": "catalog://production/lakehouse", + "persistence_reference": "storage://production/metadata-catalog", + "backup_policy_reference": "policy://production/metadata-backup-v1", + } + ) + profile["tenancy"].update( + { + "isolation_mode": "namespace_and_provider_policy", + "policy_reference": "policy://production/metadata-tenant-isolation-v1", + } + ) + profile["operations"].update( + { + "identity_owner": "team://metadata-platform", + "security_owner": "team://security-platform", + "incident_owner": "team://metadata-oncall", + "audit_log_reference": "logging://production/metadata-identity-audit", + "rotation_slo_minutes": 30, + "revocation_slo_minutes": 10, + "runbook": { + "uri": "https://runbooks.gda.internal/metadata-identity", + "version": "2026.07.28", + }, + "rollback_runbook": { + "uri": "https://runbooks.gda.internal/metadata-identity-rollback", + "version": "2026.07.28", + }, + } + ) + return profile + + +def _attestation(profile_path: Path, **changes: object) -> dict: + report = gate.build_identity_readiness_report(profile_path=profile_path, now=NOW) + profile = yaml.safe_load(profile_path.read_text(encoding="utf-8")) + payload = { + "schema": gate.ATTESTATION_SCHEMA, + "environment": gate.ENVIRONMENT, + "profile_fingerprint": report["profile_fingerprint"], + "source_revision": "c" * 40, + "observed_at": NOW.isoformat(), + "expires_at": (NOW + timedelta(days=1)).isoformat(), + "protected_environment": gate.ENVIRONMENT, + "verifier_identity": "identity://github/metadata-identity-verifier", + "evidence_uri": "https://evidence.gda.internal/metadata-identity/run-42", + "provider_versions": deepcopy(gate.EXPECTED_PROVIDERS), + "local_evidence_fingerprints": { + provider: item["evidence_fingerprint"] + for provider, item in gate.EXPECTED_LOCAL_EVIDENCE.items() + }, + "federation_fingerprint": report["federation_fingerprint"], + "provider_bindings_fingerprint": report["provider_bindings_fingerprint"], + "authorization_fingerprint": report["authorization_fingerprint"], + "tls_fingerprint": report["tls_fingerprint"], + "catalog_fingerprint": report["catalog_fingerprint"], + "tenancy_fingerprint": report["tenancy_fingerprint"], + "runbook_versions": { + name: profile["operations"][name]["version"] + for name in ("runbook", "rollback_runbook") + }, + "checks": {name: "passed" for name in gate.EXPECTED_ATTESTATION_CHECKS}, + } + payload.update(changes) + return payload + + +def test_checked_in_profile_is_valid_but_production_identity_is_blocked(): + report = gate.build_identity_readiness_report(now=NOW) + + assert report["profile_valid"] is True + assert report["profile_errors"] == [] + assert report["ready_for_protected_verification"] is False + assert report["production_identity_gate_passed"] is False + assert report["provider_minimum_privilege_verified"] is False + assert report["protected_workload_identity_verified"] is False + assert report["oidc_verified"] is False + assert report["tls_verified"] is False + assert report["production_identity_verified"] is False + assert report["production_ready"] is False + assert len(report["profile_blockers"]) == 40 + assert "federation.decision_status" in report["profile_blockers"] + assert "providers.gravitino.integration_mode" in report["profile_blockers"] + assert "catalog.gravitino_backend" in report["profile_blockers"] + assert gate.verify_report_integrity(report) == [] + + +def test_complete_profile_without_attestation_is_only_ready_for_verification(tmp_path): + profile_path = _write_profile(tmp_path, _complete_profile()) + + report = gate.build_identity_readiness_report(profile_path=profile_path, now=NOW) + + assert report["profile_valid"] is True + assert report["profile_errors"] == [] + assert report["profile_blockers"] == [] + assert report["ready_for_protected_verification"] is True + assert report["attestation_valid"] is False + assert report["production_identity_gate_passed"] is False + + +def test_fresh_bound_attestation_passes_only_the_identity_gate(tmp_path): + profile_path = _write_profile(tmp_path, _complete_profile()) + + report = gate.build_identity_readiness_report( + profile_path=profile_path, + attestation=_attestation(profile_path), + now=NOW, + ) + + assert report["attestation_valid"] is True + assert report["production_identity_gate_passed"] is True + assert report["provider_minimum_privilege_verified"] is True + assert report["protected_workload_identity_verified"] is True + assert report["oidc_verified"] is True + assert report["tls_verified"] is True + assert report["credential_rotation_verified"] is True + assert report["credential_revocation_verified"] is True + assert report["persistent_catalog_identity_binding_verified"] is True + assert report["production_identity_verified"] is True + assert report["production_ready"] is False + assert gate.verify_report_integrity(report) == [] + + +@pytest.mark.parametrize( + ("provider", "mode"), + [ + ("openmetadata", "static_jwt"), + ("gravitino", "simple"), + ("gravitino", "basic"), + ], +) +def test_profile_rejects_local_or_unvalidated_authenticators(tmp_path, provider, mode): + profile = _complete_profile() + profile["providers"][provider]["integration_mode"] = mode + + report = gate.build_identity_readiness_report( + profile_path=_write_profile(tmp_path, profile), now=NOW + ) + + assert report["profile_valid"] is False + assert f"{provider} identity integration mode is invalid" in "\n".join( + report["profile_errors"] + ) + assert report["production_identity_gate_passed"] is False + + +def test_profile_rejects_static_credentials_unpinned_component_and_http(tmp_path): + profile = _complete_profile() + profile["providers"]["gravitino"]["static_credentials_forbidden"] = False + profile["providers"]["openmetadata"]["authentication_component_reference"] = ( + "oci://registry.gda.internal/metadata/openmetadata-identity:latest" + ) + profile["tls"]["gravitino_endpoint"] = "http://localhost:8090/api" + + report = gate.build_identity_readiness_report( + profile_path=_write_profile(tmp_path, profile), now=NOW + ) + + rendered = "\n".join(report["profile_errors"]) + assert report["profile_valid"] is False + assert "static_credentials_forbidden" in rendered + assert "digest-pinned OCI" in rendered + assert "gravitino_endpoint is invalid" in rendered + + +def test_profile_rejects_authorization_and_local_evidence_drift(tmp_path): + profile = _complete_profile() + profile["authorization"]["gravitino"]["securable_objects"][1][ + "privileges" + ].append("MODIFY_TABLE") + profile["scope"]["local_evidence"]["openmetadata"][ + "evidence_fingerprint" + ] = "0" * 64 + + report = gate.build_identity_readiness_report( + profile_path=_write_profile(tmp_path, profile), now=NOW + ) + + rendered = "\n".join(report["profile_errors"]) + assert report["profile_valid"] is False + assert "minimum-privilege authorization contract" in rendered + assert "openmetadata local identity evidence binding" in rendered + + +def test_profile_rejects_sensitive_fields_and_self_asserted_claims(tmp_path): + profile = _complete_profile() + profile["federation"]["client_secret"] = "must-not-enter-profile" + profile["claims"]["production_identity_gate_passed"] = True + + report = gate.build_identity_readiness_report( + profile_path=_write_profile(tmp_path, profile), now=NOW + ) + + rendered = "\n".join(report["profile_errors"]) + assert report["profile_valid"] is False + assert "credential-bearing fields" in rendered + assert "federation inventory" in rendered + assert "may not self-assert" in rendered + + +@pytest.mark.parametrize( + "binding", + [ + "profile_fingerprint", + "federation_fingerprint", + "provider_bindings_fingerprint", + "authorization_fingerprint", + "tls_fingerprint", + "catalog_fingerprint", + "tenancy_fingerprint", + ], +) +def test_attestation_rejects_every_profile_binding_drift(tmp_path, binding): + profile_path = _write_profile(tmp_path, _complete_profile()) + attestation = _attestation(profile_path) + attestation[binding] = "0" * 64 + + report = gate.build_identity_readiness_report( + profile_path=profile_path, attestation=attestation, now=NOW + ) + + assert report["attestation_valid"] is False + assert "current profile" in "\n".join(report["attestation_errors"]) or ( + f"binding does not match: {binding}" in "\n".join(report["attestation_errors"]) + ) + + +def test_attestation_rejects_sensitive_fields_expiry_and_local_evidence_drift(tmp_path): + profile_path = _write_profile(tmp_path, _complete_profile()) + attestation = _attestation( + profile_path, + observed_at=(NOW - timedelta(days=2)).isoformat(), + expires_at=(NOW - timedelta(days=1)).isoformat(), + access_token="must-not-enter-attestation", + ) + attestation["local_evidence_fingerprints"]["gravitino"] = "0" * 64 + + report = gate.build_identity_readiness_report( + profile_path=profile_path, attestation=attestation, now=NOW + ) + + rendered = "\n".join(report["attestation_errors"]) + assert report["attestation_valid"] is False + assert "credential-bearing fields" in rendered + assert "freshness window" in rendered + assert "expired" in rendered + assert "local evidence bindings" in rendered + + +@pytest.mark.parametrize( + "check", + [ + "gravitino_administrative_deny", + "provider_direct_access_denied", + "credential_revocation", + "persistent_catalog_restart", + "cross_tenant_denial", + "rollback_rehearsal", + ], +) +def test_attestation_requires_denial_lifecycle_persistence_and_rollback(tmp_path, check): + profile_path = _write_profile(tmp_path, _complete_profile()) + attestation = _attestation(profile_path) + attestation["checks"][check] = "failed" + + report = gate.build_identity_readiness_report( + profile_path=profile_path, attestation=attestation, now=NOW + ) + + assert report["attestation_valid"] is False + assert check in "\n".join(report["attestation_errors"]) + + +def test_report_integrity_rejects_tampering_inventory_and_production_overclaim(): + report = gate.build_identity_readiness_report(now=NOW) + report["production_ready"] = True + report["oidc_verified"] = True + + errors = gate.verify_report_integrity(report) + + assert "identity readiness report fingerprint does not match" in errors + assert "identity gate may not claim overall production readiness" in errors + assert "identity gate result is inconsistent: oidc_verified" in errors + + forged = gate.build_identity_readiness_report(now=NOW) + forged["unexpected_claim"] = False + stable = { + key: value for key, value in forged.items() if key != "report_fingerprint" + } + forged["report_fingerprint"] = gate.recovery._canonical_sha256(stable) + assert "identity readiness report inventory does not match" in ( + gate.verify_report_integrity(forged) + ) + + +def test_wrapper_is_fail_closed_and_malformed_profile_is_blocked(tmp_path): + wrapper = gate.REPO_ROOT / "scripts/metadata-fabric-identity-gate.sh" + text = wrapper.read_text(encoding="utf-8") + assert "set -euo pipefail" in text + assert "metadata_fabric_identity_gate" in text + + target = tmp_path / "profile.yaml" + target.write_text("providers: [\n", encoding="utf-8") + report = gate.build_identity_readiness_report(profile_path=target, now=NOW) + assert report["profile_valid"] is False + assert report["production_identity_gate_passed"] is False + assert gate.verify_report_integrity(report) == [] + + +def test_attestation_never_records_material_in_success_report(tmp_path): + profile_path = _write_profile(tmp_path, _complete_profile()) + report = gate.build_identity_readiness_report( + profile_path=profile_path, + attestation=_attestation(profile_path), + now=NOW, + ) + rendered = json.dumps(report, sort_keys=True) + assert "client_secret" not in rendered + assert "access_token" not in rendered + assert report["production_identity_gate_passed"] is True diff --git a/docs/architecture-decisions/adr-053-production-metadata-fabric-identity-readiness-gate.md b/docs/architecture-decisions/adr-053-production-metadata-fabric-identity-readiness-gate.md new file mode 100644 index 00000000..5165840e --- /dev/null +++ b/docs/architecture-decisions/adr-053-production-metadata-fabric-identity-readiness-gate.md @@ -0,0 +1,93 @@ +# ADR-053: Production Metadata Fabric Identity Readiness Gate + +**Status**: Accepted + +**Date**: 2026-07-28 + +**Decision owners**: Metadata Platform, Security, SRE, Platform Architecture + +**Related decisions**: [ADR-019](adr-019-configuration-and-runtime-truth.md) · [ADR-046](adr-046-production-network-policy-readiness-gate.md) · [ADR-051](adr-051-local-openmetadata-bounded-provider-identity.md) · [ADR-052](adr-052-local-gravitino-basic-bounded-provider-identity.md) + +## Context + +M3-5 与 M3-6 已分别证明 OpenMetadata `1.13.1` 临时 bot 和 Gravitino `1.3.0` Basic user 的本地最小权限、越权拒绝、credential rotation/revocation 与完整清理。两次演练都依赖本地管理员 provisioner、loopback HTTP 和短生命周期凭据；Gravitino probe 还使用 memory catalog。因此它们不能证明双 provider 的生产 OIDC、受保护 workload identity、TLS、持久 catalog、tenant isolation 或生产 credential delivery。 + +对 Gravitino `1.3.0` 镜像的检查只发现 `/opt/gravitino/libs/gravitino-idp-basic-1.3.0.jar`，没有发现随镜像交付的原生 OIDC extension。生产设计不能假设不存在的 native OIDC 能力，也不能把 Basic IdP 包装成生产联邦身份。本切片需要把生产身份决策和验收条件冻结为机器可验证、默认关闭的合同，但不替团队选择 IdP、部署认证组件或制造生产证明。 + +## Options Considered + +| 方案 | 优点 | 代价/风险 | 结论 | +|---|---|---|---| +| 将 M3-5/M3-6 本地结果直接提升为生产身份 | 无新增实现 | 本地 JWT/Basic、HTTP 与 memory catalog 没有生产身份语义 | 拒绝 | +| 假设 Gravitino 内置 native OIDC | 配置看似简单 | 与 `1.3.0` 镜像内容不符，形成不可执行合同 | 拒绝 | +| 继续只在 roadmap 列举 IAM 缺口 | 无新增代码 | CI 无法拒绝 static credential、placeholder、过期证明或配置漂移 | 拒绝 | +| 版本化 pending profile + 独立、新鲜且完全绑定的 attestation | 决策与运行证据分离；可在未选型时诚实 fail closed | 需要后续 owner 明确选型并维护证明生命周期 | 采用，限定为 M3-7 | + +## Decision + +### 1. Checked-in profile 固定生产身份边界，不代表部署 + +`config/metadata-fabric-identity.production.yaml` 固定： + +- OpenMetadata `1.13.1`、Gravitino `1.3.0` 及 M3-5/M3-6 两份本地 evidence fingerprint； +- OIDC federation、`sub` workload subject、`gda_tenant_id` tenant claim、最长 900 秒 token 与 short-lived token exchange； +- 每个 provider 的 integration mode、digest-pinned authentication component、环境和 Kubernetes ServiceAccount binding，以及禁止绕过受证明身份路径； +- M3-5/M3-6 已验证的精确 allow/deny authorization contract； +- TLS 1.2+、内部 mTLS、持久 Gravitino catalog、namespace + provider policy tenant isolation； +- identity/security/incident owner、audit、rotation/revocation SLO、runbook 与 rollback runbook； +- 所有 self-reported production claim 固定为 `false`。 + +OpenMetadata 只允许 `provider_native_oidc` 或 `identity_aware_proxy`；Gravitino 只允许 `custom_oidc_authenticator` 或 `identity_aware_proxy`。`simple`、本地 `basic`、静态 JWT、密码或长生命周期 credential 都不能通过该门禁。认证组件必须是 digest-pinned OCI reference。 + +`null` 和 `decision_status=pending` 是合法的显式 blockers，所以当前 profile 可以结构有效而不假装外部决策已完成。placeholder、loopback/cluster-local 或非 HTTPS endpoint、credential-bearing 字段、mutable OCI reference、扩大后的 provider permission、provider version/local evidence 漂移或自报生产结论都会使 profile 无效。 + +### 2. 生产结论只能从受保护 attestation 派生 + +`data_agent.metadata_fabric_identity_gate` 将结果分为三层： + +1. `profile_valid`：profile 的结构、安全边界和本地 evidence binding 可信； +2. `ready_for_protected_verification`：federation、双 provider、TLS、catalog、tenancy 和 operations 的 40 项外部输入已经明确； +3. `production_identity_gate_passed`：另有新鲜的 production protected-environment attestation，且绑定当前 profile、source revision、provider versions、本地 evidence、六组派生 fingerprint 和两个 runbook version。 + +attestation 的精确 18 项检查必须全部为 `passed`：OIDC discovery/token exchange、workload subject 和 tenant claim binding、双 provider allow/administrative deny、provider direct-access bypass denial、static credential absence、rotation/revocation、TLS/mTLS、持久 catalog restart、cross-tenant denial、audit delivery 和 rollback rehearsal。观测时间不得早于验证时刻 24 小时，expiry 必须在未来且有效期最长七天，evidence URI 必须是非本地 HTTPS；任何漂移、过期或失败都关闭门禁。 + +### 3. 身份门禁通过也不等于平台生产就绪 + +同一有效 attestation 可唯一派生 provider minimum privilege、protected workload identity、OIDC、TLS、rotation/revocation、persistent catalog identity binding 和 production identity claims。报告中的 `production_ready` 始终固定为 `false`；生产 recovery、observability、NetworkPolicy、upgrade、registry provenance、持久 binding、ingestion/OpenLineage conformance 等退出门仍须独立通过。 + +`validate` 只验证 checked-in profile，因此有效的 pending contract 在 CI 中成功；`evaluate` 必须提供 attestation，且仅在身份门禁实际通过时成功；`verify` 拒绝 fingerprint 漂移、派生 claim 不一致和 overall production overclaim。本切片不提交真实 attestation、不部署认证组件，也不修改 provider。 + +## Verification + +当前 checked-in profile： + +- profile fingerprint：`2e9d5cac3560b853820f923669f6794ead63bcb36a528639fc0e9539e148ee2f`； +- report fingerprint：`c607589ee25a87acc8a1ab71372618a9a4c10c1e8ebff15b8db7e78b37600b9f`； +- `profile_valid=true` 且无 profile errors； +- 40 项 federation/provider/TLS/catalog/tenancy/operations 外部输入以 blockers 暴露； +- `ready_for_protected_verification=false`、`attestation_valid=false`； +- 全部生产身份 claims 与 `production_ready` 固定为 `false`。 + +26 个定向测试覆盖 pending/complete profile、新鲜且完全绑定的合成 attestation、两类 provider integration mode、Gravitino native OIDC/Basic/simple 拒绝、authorization/local evidence/binding drift、placeholder/HTTP/loopback/mutable OCI、敏感字段、自报 claim、过期或失败证明、报告篡改和生产 overclaim。 + +## Claim Boundary + +允许声明： + +- M3-7 production Metadata Fabric identity readiness contract 已建立； +- 当前 pending profile 结构有效，并机器可读地暴露 40 个 blockers； +- 合成完整 profile/attestation 验证了 fail-closed 门禁逻辑。 + +当前不得声明： + +- 已选择、部署或验证生产 IdP、OIDC federation、identity proxy 或 Gravitino custom authenticator； +- 已验证生产 workload identity、TLS/mTLS、tenant isolation、持久 catalog identity binding 或双 provider minimum privilege； +- `production_identity_gate_passed=true` 或 `production_ready=true`。 + +## Consequences + +**Positive**：本地 Basic/JWT evidence 不再可能被误读为生产身份；Gravitino 的实际扩展边界被显式记录；受保护环境可用同一 profile-bound gate 发现 credential、权限、配置和证据漂移。 + +**Negative**：M3-7 本身不增加登录或授权能力。门禁会保持 blocked，直到 Metadata Platform、Security 和 SRE 完成 IdP、provider integration、持久 catalog、tenant、TLS 与运营决策。 + +**Next gate**：批准并物化 production profile，在受保护环境部署 digest-pinned authentication path，生成绑定当前 source/profile 的真实 attestation并通过全部 18 项检查；随后将该身份路径用于受控双 provider ingestion、持久 binding 和 production OpenLineage conformance，且继续独立完成其余 production gates。 diff --git a/docs/roadmap-ar0-platform-truth-2026-07-24.md b/docs/roadmap-ar0-platform-truth-2026-07-24.md index 0d72c035..6bc69e1d 100644 --- a/docs/roadmap-ar0-platform-truth-2026-07-24.md +++ b/docs/roadmap-ar0-platform-truth-2026-07-24.md @@ -199,7 +199,7 @@ Temporal 继续保持目标组件状态，不在这一包并行接入。OpenMeta 当前完成仅指本地合同、授权 evidence、outbox/callback 代码、数据库成功终局门、托管 worker 代码、默认关闭的部署模板及离线 activation/release preflight、candidate/registry/provenance/artifact-release/live observation evidence gate、合成 golden slice、定向测试、真实 PostgreSQL 16 事务边界和 canonical mainline 治理。`candidate_validated`、`registry_subject_bound`、本地合成 `provenance_verified`、`ready_for_activation`、`ready_for_staging_apply`、`verified_for_staging_apply` 和本地 live collection 都不等于真实镜像已 attested 或 staging 已部署；真实 IAM/OIDC 与 service token 生命周期、首次 GHCR publish/verify、真实 provenance artifact verify、registry-backed live staging revision、worker/callback 扩容运行、golden slice staging 运行链、受保护 release/live evidence provenance、独立 DolphinScheduler metadata PostgreSQL 和真实数据终局证据仍属于 4.7 后续切片。 -### 4.8 Metadata Fabric Bridge M1 + M2 + M3-6（本地双 provider 受限身份已分别验证） +### 4.8 Metadata Fabric Bridge M1 + M2 + M3-7（生产双 provider 身份验收合同已冻结） 第八块回到 AR-1 的 metadata control plane，以 [ADR-036](architecture-decisions/adr-036-read-only-metadata-fabric-bridge-contract.md) 固定 OpenMetadata + Gravitino + GDA Control Ledger 的首条 table slice： @@ -226,8 +226,9 @@ Temporal 继续保持目标组件状态，不在这一包并行接入。OpenMeta 21. [ADR-050](architecture-decisions/adr-050-idempotent-openlineage-http-delivery.md) 已新增 migration 098 与 tenant-scoped lineage outbox；Gateway 只有在 M3-3 binding、execution-plan 和完整 M3-1 source plan 精确匹配后才可 enqueue。真实 loopback HTTP 演练让接收端先提交事件再返回 503，第二次以同一 `Idempotency-Key` 重发并返回 duplicate 200；共 2 个 wire requests、1 次唯一接受、最终 2 attempts/delivered，完成项不再 claim。delivery UUID 为 `49a54408-b3a8-5843-a27d-6395c080af99`，event SHA 为 `4929e51c4126e09415a9fc1578c9401077c5d7c374294e70deeebd29c8216dd2`，evidence SHA 为 `8fa87a34a39b900df0673f11d0301c9f5155ce64ff9502125478ec59a3f0fdb6`。该结论是本地 `at_least_once_with_receiver_idempotency`，不是网络 exactly-once 或生产 receiver 证明。 22. [ADR-051](architecture-decisions/adr-051-local-openmetadata-bounded-provider-identity.md) 已在 OpenMetadata `1.13.1` 创建临时非管理员 bot，effective roles 精确为 provider 强制 `DefaultBotRole` 加项目专用 role；项目新增 policy 仅允许 `table/Create`。bot 创建/read-back 临时 table 分别返回 201/200，创建 policy 返回 403；JWT 轮换后旧值返回 401，新值返回 200，吊销后新值返回 401。table、bot、user、role、policy 与 denial probe 最终均不存在，evidence SHA 为 `61b6a3429ae948f563bfc2bd012d8b586be581704cec646fd5e74b991243f03f`。该 scoped local 结论不包含 Gravitino、OIDC、Kubernetes-to-provider identity exchange 或生产 credential delivery。 23. [ADR-052](architecture-decisions/adr-052-local-gravitino-basic-bounded-provider-identity.md) 已在隔离 Gravitino `1.3.0` namespace 启用 Basic IdP 与 authorization，bounded user 只获得 `lakehouse` 的 `USE_CATALOG` 以及 `lakehouse.published` 的 `USE_SCHEMA`/`CREATE_TABLE`；table create/read 为 200/200，越权 catalog create 为 403，密码轮换使旧值返回 401，IdP 用户删除使替换值返回 401。临时 metalake/catalog/schema/table/user/role、namespace 与 loopback port-forward 均已清理，证据 SHA 为 `f0b0de1f80f079d43318937e0a0cc151a8546e9e307bef204738b1367f9b29fd`。这只是 Gravitino Basic local POC；probe catalog 使用 memory backend，不包含 OIDC、TLS、Kubernetes-to-provider identity exchange、持久生产 catalog 或双 provider production identity。 +24. [ADR-053](architecture-decisions/adr-053-production-metadata-fabric-identity-readiness-gate.md) 已将 OIDC federation、双 provider integration、digest-pinned authentication component、workload/tenant claim、Kubernetes ServiceAccount、禁止 direct bypass、short-lived token、M3-5/M3-6 精确 allow/deny contract、TLS/mTLS、持久 Gravitino catalog、tenant isolation、owner/audit/SLO/runbook 与 18 项 protected attestation check 冻结为 fail-closed profile。Gravitino `1.3.0` 镜像只发现 Basic IdP jar，因此生产只允许明确选择并证明 `custom_oidc_authenticator` 或 `identity_aware_proxy`，不假设 native OIDC。当前 profile fingerprint 为 `2e9d5cac3560b853820f923669f6794ead63bcb36a528639fc0e9539e148ee2f`，合同有效但 40 项外部输入 blocked，没有真实 attestation，全部 production identity claims 仍为 `false`。 -此处 M1 只证明静态合同和只读 HTTP 边界；M2a 只证明本地 live foundation 与 PVC 重挂载连续性；M2b-1/M2b-2 分别限定在同集群新 PVC 和同集群隔离 repository；M2b-3 的 `local_cross_cluster_recovery_verified=true` 只限定在 `local_same_host_distinct_kubernetes_clusters_external_s3_repository`；M2c-1/M2c-2/M2c-3 分别限定本地 provider metrics、临时双周期 OTel 和单 job scrape recovery；M2c-4/M2d-2 只证明 production observability/NetworkPolicy profile 与 attestation 合同可校验；M2d-1 只证明本地两节点 kindnet 的隔离合成流量；M3-1 的 terminal evidence 与 M3-2 的 PolicyDecision/Approval 仍是 deterministic local fixtures。M3-2 只把 projection 写入本地 provider 并证明 retained target 的单次零写入 replay；M3-3 只把该本地 evidence 对应的 binding 写入临时 GDA Control 账本；M3-4 只向无认证 loopback receiver 发送精确 candidate 并验证 503 后幂等恢复；M3-5 只证明 OpenMetadata 在 provider 强制默认 role 之上的项目新增 grant 限定为 `table/Create`，以及本地 JWT 轮换/吊销和越权拒绝；M3-6 只证明隔离 Gravitino Basic IdP 的 bounded table-create、catalog-create 拒绝、登录轮换/吊销和完整清理。M3-2 ingestion 仍使用 bootstrap admin，Gravitino M3-6 probe catalog 使用 memory backend，restart persistence 未验证。生产持久 binding、ResourceVersion 和 legacy authority 都未写入；双 provider/生产最小权限、protected workload identity、OIDC、TLS、持久 catalog、tenant isolation、真实 receiver/alert/SLO、受保护 provider policy、生产 OpenLineage、生产 ingest/conformance、两项 production gate 和 `production_ready` 仍为 `false`。 +此处 M1 只证明静态合同和只读 HTTP 边界；M2a 只证明本地 live foundation 与 PVC 重挂载连续性；M2b-1/M2b-2 分别限定在同集群新 PVC 和同集群隔离 repository；M2b-3 的 `local_cross_cluster_recovery_verified=true` 只限定在 `local_same_host_distinct_kubernetes_clusters_external_s3_repository`；M2c-1/M2c-2/M2c-3 分别限定本地 provider metrics、临时双周期 OTel 和单 job scrape recovery；M2c-4/M2d-2 只证明 production observability/NetworkPolicy profile 与 attestation 合同可校验；M2d-1 只证明本地两节点 kindnet 的隔离合成流量；M3-1 的 terminal evidence 与 M3-2 的 PolicyDecision/Approval 仍是 deterministic local fixtures。M3-2 只把 projection 写入本地 provider 并证明 retained target 的单次零写入 replay；M3-3 只把该本地 evidence 对应的 binding 写入临时 GDA Control 账本；M3-4 只向无认证 loopback receiver 发送精确 candidate 并验证 503 后幂等恢复；M3-5 只证明 OpenMetadata 在 provider 强制默认 role 之上的项目新增 grant 限定为 `table/Create`，以及本地 JWT 轮换/吊销和越权拒绝；M3-6 只证明隔离 Gravitino Basic IdP 的 bounded table-create、catalog-create 拒绝、登录轮换/吊销和完整清理；M3-7 只证明 pending production identity profile、profile-bound attestation 和派生 claim 的 fail-closed 合同可校验，没有部署或证明真实身份路径。M3-2 ingestion 仍使用 bootstrap admin，Gravitino M3-6 probe catalog 使用 memory backend，restart persistence 未验证。生产持久 binding、ResourceVersion 和 legacy authority 都未写入；双 provider/生产最小权限、protected workload identity、OIDC、TLS、持久 catalog、tenant isolation、真实 receiver/alert/SLO、受保护 provider policy、生产 OpenLineage、生产 ingest/conformance、三项 production gate 和 `production_ready` 仍为 `false`。 ## 5. 重新评估条件 diff --git a/docs/roadmap.md b/docs/roadmap.md index 9535f9d2..76bea67f 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -417,7 +417,7 @@ AR-0 Architecture / Schema / Runtime Truth - Web、DolphinScheduler、OpenMetadata、Gravitino、worker 和 bridge 重启后不丢版本、产品、血缘、PlatformRun correlation 或审批状态。 - DataOps 状态可从 DolphinScheduler/OpenMetadata/Gravitino/GDA evidence 恢复；任何 release/deployment 都能从事件、Artifact、评测、策略、incident 和 rollback pointer 重放。Temporal durable recovery 在 AR-5/AR-7 单独验收。 -当前 Metadata Fabric 证据边界：M1 只读 bridge 合同已验证；ADR-037 至 ADR-046 分别覆盖本地 foundation/recovery/metrics/network-policy 演练与 production readiness contracts；ADR-047 至 ADR-050 已依次建立 deterministic projection plan、本地双 provider replay、tenant-scoped binding ledger 与本地 OpenLineage 幂等 wire delivery；ADR-051 以临时非管理员 OpenMetadata bot 证明项目专用 grant 只有 `table/Create`、`policy/Create` 被 403 拒绝，且 JWT 轮换/吊销后旧值/当前值均返回 401；ADR-052 又在隔离 Gravitino `1.3.0` Basic IdP 中证明 bounded user 的 `USE_CATALOG`、`USE_SCHEMA`、`CREATE_TABLE` 范围、catalog-create 403、密码轮换和用户吊销。`local_openmetadata_minimum_privilege_verified=true` 与 `local_gravitino_minimum_privilege_verified=true` 都只描述各自临时 provider rehearsal；M3-2 ingestion 仍使用 bootstrap admin，Gravitino probe catalog 仍是 memory catalog。因此 `provider_minimum_privilege_verified`、protected workload identity、OIDC、TLS、持久 catalog、生产 ingestion/conformance、生产 lineage receiver 与 `production_ready` 仍为 false。 +当前 Metadata Fabric 证据边界：M1 只读 bridge 合同已验证；ADR-037 至 ADR-046 分别覆盖本地 foundation/recovery/metrics/network-policy 演练与 production readiness contracts；ADR-047 至 ADR-050 已依次建立 deterministic projection plan、本地双 provider replay、tenant-scoped binding ledger 与本地 OpenLineage 幂等 wire delivery；ADR-051 以临时非管理员 OpenMetadata bot 证明项目专用 grant 只有 `table/Create`、`policy/Create` 被 403 拒绝，且 JWT 轮换/吊销后旧值/当前值均返回 401；ADR-052 又在隔离 Gravitino `1.3.0` Basic IdP 中证明 bounded user 的 `USE_CATALOG`、`USE_SCHEMA`、`CREATE_TABLE` 范围、catalog-create 403、密码轮换和用户吊销；ADR-053 将生产 OIDC federation、双 provider integration/workload identity、最小权限、TLS/mTLS、持久 Gravitino catalog、tenant isolation、运营责任和新鲜 protected attestation 冻结为 fail-closed readiness contract。Gravitino `1.3.0` 镜像只发现 Basic IdP，不假设 native OIDC；当前 profile 仍有 40 个外部 blockers 且未提交真实 attestation。`local_openmetadata_minimum_privilege_verified=true` 与 `local_gravitino_minimum_privilege_verified=true` 都只描述各自临时 provider rehearsal；M3-2 ingestion 仍使用 bootstrap admin，Gravitino probe catalog 仍是 memory catalog。因此 `provider_minimum_privilege_verified`、protected workload identity、OIDC、TLS、持久 catalog、生产 ingestion/conformance、生产 lineage receiver、`production_identity_gate_passed` 与 `production_ready` 仍为 false。 ### AR-2 — Source, Ingestion and Geospatial Lakehouse Vertical Slice（P0） diff --git a/docs/system-of-record-matrix-2026-07-24.md b/docs/system-of-record-matrix-2026-07-24.md index 59de5474..8180b267 100644 --- a/docs/system-of-record-matrix-2026-07-24.md +++ b/docs/system-of-record-matrix-2026-07-24.md @@ -2,9 +2,9 @@ 日期：2026-07-28 -阶段：AR-0 `in_progress`；AR-1 gateway、成功终局 evidence gate、DolphinScheduler adapter sandbox POC、Metadata Fabric M1/M2、M2c-4/M2d-2 production readiness contracts、M3-1/M3-2、M3-3 local binding ledger、M3-4 local OpenLineage wire delivery、M3-5 local OpenMetadata bounded identity 与 M3-6 local Gravitino Basic bounded identity 已验证，生产 provider ingestion、生产观测、生产 policy/tenant isolation 和生产切换仍 `in_progress` +阶段：AR-0 `in_progress`；AR-1 gateway、成功终局 evidence gate、DolphinScheduler adapter sandbox POC、Metadata Fabric M1/M2、M2c-4/M2d-2 production readiness contracts、M3-1/M3-2、M3-3 local binding ledger、M3-4 local OpenLineage wire delivery、M3-5 local OpenMetadata bounded identity、M3-6 local Gravitino Basic bounded identity 与 M3-7 production identity readiness contract 已验证，生产 provider ingestion、生产观测、生产 policy/tenant isolation、生产 identity attestation 和生产切换仍 `in_progress` -适用分支：`feat/ar1-metadata-fabric-gravitino-identity` +适用分支：`feat/ar1-metadata-fabric-identity-readiness-gate` ## 判定规则 @@ -25,8 +25,8 @@ | 湖仓表与 snapshot | Iceberg/STAC/S3A 有局部实现，尚无通用发布权威 | STAC item、GeoParquet export | Iceberg catalog snapshot 是分析表版本权威；对象是物理内容，STAC 是发现投影 | Data Platform | AR-2 | | 在线空间数据 | PostGIS 业务表是当前编辑/查询事实，部分临时表混入 | Martin MVT、API JSON、导出文件 | 已批准 DataProductVersion 物化到 PostGIS；不能由瓦片或临时表反向定义产品版本 | GIS/Data Platform | AR-2 -> AR-4 | | 数据资产身份与版本 | `gda_control.resource/resource_version` 已实现 identity、hash、predecessor、tenant FK 和幂等 gateway 写入；`agent_data_assets`、`agent_asset_versions` 仍是兼容写路径 | UI catalog、search index、STAC | GDA ledger 管身份与版本绑定；旧行只有在 tenant、authority identity、checksum 和 version evidence 完整时才可形成 eligible plan；OpenMetadata 管治理目录，Gravitino 管技术对象映射 | Metadata Platform | AR-1 gateway 已验证 -> 生产切换待验收 | -| 技术元数据 | M1 已冻结 Gravitino table ref/reconciliation；M2 已验证本地 foundation/recovery/metrics/policy 和 production readiness contracts；M3-1 固定 technical projection intent，M3-2 已在 Gravitino memory catalog 创建/read-back，M3-3 将验证后的 ref 追加到 tenant-scoped 本地 binding ledger；M3-6 又在隔离 Gravitino Basic IdP 中验证 bounded table-create、catalog-create 拒绝、密码轮换/用户吊销和完整清理 | harvester 结果、合成 response、本地 sandbox/recovery/metrics/policy/ingestion/identity observation、projection plan、provider evidence、binding ledger 与 readiness report | 源系统技术对象是原始证据；Gravitino 映射并联邦，不能覆盖业务 ResourceVersion；GDA binding ledger 只记录已验证关系，本地 memory catalog/evidence 不得冒充生产持久技术权威；M3-6 的 Basic IdP 和 loopback HTTP 也不是生产身份 | Metadata Platform | AR-1 M1/M2 + M3-6 local identity 已验证 -> 认证持久 catalog/受保护 workload identity/生产 conformance 待执行 | -| 治理目录 | M1 已冻结 OpenMetadata table ref/reconciliation；M2 已验证本地 foundation/recovery/metrics/policy 和 production readiness contracts；M3-2 已用 bootstrap admin 创建目标并回读真实 UUID；M3-3 将该 UUID 经 evidence gate 追加到本地 GDA binding ledger；M3-4 将精确 OpenLineage candidate 经 outbox 投递到本地 HTTP receiver；M3-5 已验证临时非管理员 bot 的 scoped `table/Create` grant、policy-create 拒绝及 JWT 轮换/吊销 | 搜索/页面视图、合成 response、本地 sandbox/recovery/metrics/policy/ingestion/identity observation、projection/provider evidence、binding ledger、lineage outbox/receipt 与 OpenLineage event | OpenMetadata 为 owner/glossary/classification/quality discoverability 权威；GDA ledger 保留审批/provider identity，outbox 只拥有投递状态，receiver 拥有接收状态；均不反写 ResourceVersion | Governance | AR-1 M1/M2 + M3-5 local bounded identity 已验证 -> protected identity ingestion/生产持久 binding/受保护 production receiver 待执行 | +| 技术元数据 | M1 已冻结 Gravitino table ref/reconciliation；M2 已验证本地 foundation/recovery/metrics/policy 和 production readiness contracts；M3-1 固定 technical projection intent，M3-2 已在 Gravitino memory catalog 创建/read-back，M3-3 将验证后的 ref 追加到 tenant-scoped 本地 binding ledger；M3-6 又在隔离 Gravitino Basic IdP 中验证 bounded table-create、catalog-create 拒绝、密码轮换/用户吊销和完整清理；M3-7 已冻结 production identity profile/attestation gate | harvester 结果、合成 response、本地 sandbox/recovery/metrics/policy/ingestion/identity observation、projection plan、provider evidence、binding ledger 与 readiness report | 源系统技术对象是原始证据；Gravitino 映射并联邦，不能覆盖业务 ResourceVersion；GDA binding ledger 只记录已验证关系，本地 memory catalog/evidence 不得冒充生产持久技术权威；Basic IdP、loopback HTTP、pending profile 和合成 attestation 都不是生产身份 | Metadata Platform | AR-1 M1/M2 + M3-6 local identity + M3-7 readiness contract 已验证 -> 认证持久 catalog/受保护 attestation/生产 conformance 待执行 | +| 治理目录 | M1 已冻结 OpenMetadata table ref/reconciliation；M2 已验证本地 foundation/recovery/metrics/policy 和 production readiness contracts；M3-2 已用 bootstrap admin 创建目标并回读真实 UUID；M3-3 将该 UUID 经 evidence gate 追加到本地 GDA binding ledger；M3-4 将精确 OpenLineage candidate 经 outbox 投递到本地 HTTP receiver；M3-5 已验证临时非管理员 bot 的 scoped `table/Create` grant、policy-create 拒绝及 JWT 轮换/吊销；M3-7 将其 allow/deny 范围纳入双 provider production identity gate | 搜索/页面视图、合成 response、本地 sandbox/recovery/metrics/policy/ingestion/identity observation、projection/provider evidence、binding ledger、lineage outbox/receipt、OpenLineage event 与 readiness report | OpenMetadata 为 owner/glossary/classification/quality discoverability 权威；GDA ledger 保留审批/provider identity，outbox 只拥有投递状态，receiver 拥有接收状态；pending identity profile 和合成 attestation 均不反写 ResourceVersion 或建立生产权威 | Governance | AR-1 M1/M2 + M3-5 local bounded identity + M3-7 readiness contract 已验证 -> protected identity ingestion/生产持久 binding/受保护 production receiver 待执行 | | 血缘 | `gda_control.lineage_event` 已实现 immutable version edge 和幂等 gateway ingest；`agent_asset_lineage` 旧记录仍是可变 asset edge | OpenMetadata lineage graph、UI DAG | 只有 source/target ResourceVersion 与 event checksum 证据完整的旧记录可形成 eligible plan；目录图只作可重建投影 | Data Platform | AR-1 gateway 已验证 -> adapter 待接入 | | Definition | `gda_control.platform_definition_version` 已绑定 definition ResourceVersion、完整逻辑 hash 和原子 gateway registration；3.4.2 adapter 可编译、创建并上线 provider DAG；binding 已以 append-only `execution_plan` Artifact 持久化并可按 tenant + artifact UUID 读取，旧 workflow/template/YAML 仍在写入 | 编辑器状态、DolphinScheduler DAG/definition | 旧 workflow 必须规范化并完整 hash 后才可形成 PlatformDefinitionVersion；provider binding 作为 ExecutionPlanArtifact/evidence，不可反写 definition | DataOps | AR-1 binding persistence 代码已验证 -> staging 调用链待验收 | | Run 最终状态 | `gda_control.platform_run/event` 已实现受控 submit/read/CAS；通用 transition 已禁止 `succeeded`，专用数据库 finalizer 只接受精确 workload、DolphinScheduler success observation、内容匹配 output、独立 passed QualityResult/evidence 和 input-to-output lineage；adapter standalone API path 已验证，但端到端 staging 尚未完成，legacy 路径继续运行 | Redis progress、日志、DolphinScheduler state、attempt observation | 旧 run 到 PlatformRun 永久 prohibited；已有 PlatformRun correlation 时才可转为 observation；provider 终态只进入 `reconciling`，ledger 经证据门唯一裁决成功 | DataOps/AgentOps | AR-1 success authority 本地/PostgreSQL 已验证 -> staging/生产切换待验收 | @@ -34,7 +34,7 @@ | 事件交付 | Standards outbox 已数据库耐久；`gda_control.platform_command_outbox` 已为 DolphinScheduler dispatch/reconcile 提供 tenant RLS、lease claim、幂等 callback、薄 consumer library 和 managed worker process；其他 WebSocket/bot/feedback 多为 best effort | command delivery status、消费者 claim、worker status JSON、WebSocket 消息 | command/event 与源事实同事务入 outbox，幂等 consumer 交付；worker status、outbox 状态、缓存或 socket 都不是 Run/业务权威 | Platform/Integrations | AR-1 command delivery/worker 代码已验证 -> staging worker/callback 待部署 | | 质量结果 | `gda_control.quality_result` 已提供 tenant RLS、append-only gateway 写入，绑定 Run、output ResourceVersion、rule version、verdict、metrics、evidence Artifact 和独立 evaluator；standards、QC、MMFE 专项结果仍未迁移 | dashboard、OpenMetadata quality summary | GDA ledger 保存产品终局所需的不可变 verdict/evidence；OpenMetadata 与 UI 只作可重建发现投影；旧结果缺稳定版本和证据时不得升级为终局依据 | Governance/DataOps | AR-1 最小成功证据已验证 -> 真实规则/staging 待接入 | | 标准与语义定义 | `std_*`、semantic registry 和 YAML 共同存在，生命周期未统一 | prompt/context、搜索索引 | 版本化 Standard/SemanticDefinition 经审批后为权威；Agent context 只消费批准版本 | Governance | AR-1 -> AR-3 | -| 身份与权限 | Chainlit user 可显式绑定 tenant；versioned API 从认证 principal 派生 SubjectContext；`gda_control_gateway` 是 non-login/non-bypass 最小权限角色；Run 可引用强类型 PolicyDecision/Approval Artifact；M3-5 本地 OpenMetadata bot 和 M3-6 隔离 Gravitino Basic user 分别验证 scoped grant、越权拒绝和 credential rotation/revocation，但均由本地管理员临时 provision，Gravitino 使用 loopback HTTP | session/cache、前端菜单权限、本地 provider identity evidence | IdP/workload identity 提供真实 service identity；PolicyDecision/Approval 继续绑定不可变资源与 execution plan；完成受保护 OIDC/IAM provisioning、双 provider 最小权限、secret delivery、TLS 和轮换/吊销后才能生产切换 | Security | AR-1 local provider identities 已分别验证 -> protected 双 provider IAM 待验收 | +| 身份与权限 | Chainlit user 可显式绑定 tenant；versioned API 从认证 principal 派生 SubjectContext；`gda_control_gateway` 是 non-login/non-bypass 最小权限角色；Run 可引用强类型 PolicyDecision/Approval Artifact；M3-5/M3-6 分别验证本地 provider scoped grant、越权拒绝和 credential rotation/revocation；M3-7 已冻结生产 OIDC/workload/tenant binding、TLS、持久 catalog 与 attestation contract，但 40 个外部输入仍 blocked | session/cache、前端菜单权限、本地 provider identity evidence、pending profile 与合成 readiness report | IdP/workload identity 提供真实 service identity；PolicyDecision/Approval 继续绑定不可变资源与 execution plan；只有 fresh protected attestation 可派生双 provider production identity claims，profile、Basic/JWT evidence 或人工批准均不可替代 | Security | AR-1 local identities + production readiness contract 已验证 -> protected 双 provider IAM/attestation 待执行 | | GIS 服务定义与 active revision | Martin、REST/MVT/STAC endpoints 和配置直接暴露 | Ingress、tile cache、客户端图层 | GIS Service Control Plane 管 Service/Layer/Style/TMS/DeploymentRevision；provider/Gateway 仅执行 | GIS Platform | AR-4 | | 缓存与进度 | Redis、进程内 dict/task map | UI progress、tile/context cache | 永不作为资产、Run、workflow 或产品权威；丢失后必须从 ledger/provider 重建 | SRE/Platform | 持续约束 | | Agent/Prompt/Model bundle | 多个 registry 与 YAML 存在，尚无统一 deployment revision | trace、eval dashboard | AgentSpecBundle + EvaluationBinding + DeploymentRevision；只消费已发布 DataProductVersion | AgentOps | AR-5 | @@ -56,7 +56,7 @@ 11. `platform_command_outbox` 只拥有投递状态；callback 只触发 reconcile，不能把 provider payload 直接写成 PlatformRun 状态或平台终局。 12. QualityResult evaluator 必须是 workload，且成功终局中的 evaluator 不能等于 Run workload；该代码级职责分离不替代生产 IAM。 13. `candidate_validated`、`registry_subject_bound`、GitHub provenance action 成功、CI artifact、离线 preflight、未独立 attested 的 live observation JSON 或人工批准都不能单独授权 production；缺少同一 source revision 的 OCI subject 独立验证、registry/live revision/identity/health/golden-slice 绑定及受保护 provenance 时，promotion 必须失败。 -14. Metadata Fabric M1 只允许 OpenMetadata/Gravitino GET；M2 只执行本地 foundation/recovery/metrics/policy 演练或验证 production readiness profile；M3-1 只从 synthetic terminal evidence 生成 plan/candidate；M3-2 只允许 exact local PolicyDecision/Approval 后向本地 provider 写 projection；M3-3 只将同一 source evidence 经 PlatformGateway 写入临时 append-only binding ledger；M3-4 只经 tenant-scoped outbox 向无认证 loopback receiver 投递精确 candidate，并验证 at-least-once + receiver idempotency；M3-5 只证明临时 OpenMetadata bot 在 provider 强制 `DefaultBotRole` 之上的项目新增 grant 是 `table/Create`，并验证 policy-create 拒绝与本地 JWT 轮换/吊销；M3-6 只证明隔离 Gravitino Basic user 的 bounded table-create、catalog-create 拒绝、密码轮换/用户吊销和完整清理。本地 bootstrap provisioner、Basic IdP、loopback HTTP、memory catalog、临时 identity/ledger/outbox、loopback receiver、pending profile、合成 attestation 和 local evidence 都不等于双 provider/生产最小权限、protected workload identity/OIDC、持久 catalog/binding、TLS、受保护 OpenLineage receiver、tenant isolation、alert/SLO、生产 ingestion/conformance 或生产写权威。 +14. Metadata Fabric M1 只允许 OpenMetadata/Gravitino GET；M2 只执行本地 foundation/recovery/metrics/policy 演练或验证 production readiness profile；M3-1 只从 synthetic terminal evidence 生成 plan/candidate；M3-2 只允许 exact local PolicyDecision/Approval 后向本地 provider 写 projection；M3-3 只将同一 source evidence 经 PlatformGateway 写入临时 append-only binding ledger；M3-4 只经 tenant-scoped outbox 向无认证 loopback receiver 投递精确 candidate，并验证 at-least-once + receiver idempotency；M3-5 只证明临时 OpenMetadata bot 在 provider 强制 `DefaultBotRole` 之上的项目新增 grant 是 `table/Create`，并验证 policy-create 拒绝与本地 JWT 轮换/吊销；M3-6 只证明隔离 Gravitino Basic user 的 bounded table-create、catalog-create 拒绝、密码轮换/用户吊销和完整清理；M3-7 只冻结 production identity profile、精确 attestation binding 与 fail-closed 派生 claims，既不部署 identity path，也不提交真实 production attestation。Gravitino `1.3.0` Basic IdP 不算 OIDC，生产必须明确选择并证明 custom OIDC authenticator 或 identity-aware proxy。本地 bootstrap provisioner、Basic IdP、loopback HTTP、memory catalog、临时 identity/ledger/outbox、loopback receiver、pending profile、合成 attestation 和 local evidence 都不等于双 provider/生产最小权限、protected workload identity/OIDC、持久 catalog/binding、TLS、受保护 OpenLineage receiver、tenant isolation、alert/SLO、生产 ingestion/conformance 或生产写权威。 ## 已建立的 AR-0/AR-1 entry 证据 @@ -85,6 +85,8 @@ - Metadata Fabric M3-3 已新增 tenant-scoped append-only binding ledger，将 M3-2 的真实 OpenMetadata UUID、Gravitino ref、execution-plan、PolicyDecision、Approval 与 provider evidence 经 PlatformGateway 精确校验后落账；空临时 PostgreSQL 首次提交 `created=true`、replay `created=false`，跨租户读取、UPDATE/DELETE 均拒绝。binding SHA 为 `125d7197f05ff9c37999a94d090d123dcf905480b776da0738d9625ab5045598`，record SHA 为 `19bdbddedc27d2ed8a35119e8f065a47a02345f9bbd3a51075856cb9587f4176`，evidence SHA 为 `518bfed363aba34e539ada19ea1dc708bacc9eba6578ccab165d11bccfc05223`。该演练不调用 provider，`provider_minimum_privilege_verified=false`、`oidc_verified=false`、`durable_catalog_verified=false`、`live_openlineage_emission_verified=false`、`production_ingestion_verified=false`、`production_ready=false`。 - Metadata Fabric M3-4 已新增 tenant-scoped lineage outbox、Gateway claim/complete/fail 和严格 loopback HTTP emitter；接收端第一次提交 event 后返回 503，第二次以同一 idempotency key/body 重发并只返回 duplicate 200，形成 2 wire requests、1 unique accept、2 attempts 和不可再 claim 的 delivered receipt。delivery UUID 为 `49a54408-b3a8-5843-a27d-6395c080af99`，event SHA 为 `4929e51c4126e09415a9fc1578c9401077c5d7c374294e70deeebd29c8216dd2`，evidence SHA 为 `8fa87a34a39b900df0673f11d0301c9f5155ce64ff9502125478ec59a3f0fdb6`。`local_wire_openlineage_delivery_verified=true` 仅表示真实本机 HTTP wire；`provider_minimum_privilege_verified=false`、`oidc_verified=false`、`durable_catalog_verified=false`、`production_receiver_verified=false`、`production_ingestion_verified=false`、`production_ready=false`。 - Metadata Fabric M3-5 已创建临时 OpenMetadata 非管理员 bot，effective roles 只有 provider 强制 `DefaultBotRole` 与项目 role，项目 policy 只有 `table/Create`；table create/read 为 201/200，policy create 为 403，旧 JWT 轮换后为 401，新 JWT 吊销后为 401，六项 provider cleanup 与 port-forward cleanup 全部通过。evidence SHA 为 `61b6a3429ae948f563bfc2bd012d8b586be581704cec646fd5e74b991243f03f`。`local_openmetadata_minimum_privilege_verified=true` 只描述强制 default role 之上的项目新增 grant；`provider_minimum_privilege_verified=false`、`protected_workload_identity_verified=false`、`oidc_verified=false`、`gravitino_authentication_verified=false`、`production_identity_verified=false`、`production_ready=false`。 +- Metadata Fabric M3-6 已在隔离 Gravitino `1.3.0` Basic IdP 中将 bounded user 限定为 `lakehouse` 的 `USE_CATALOG` 与 `lakehouse.published` 的 `USE_SCHEMA`/`CREATE_TABLE`；table create/read 为 200/200，catalog create 为 403，旧密码轮换后为 401，用户删除后替换密码为 401，临时 provider 对象、namespace 和 port-forward 全部清理。evidence SHA 为 `f0b0de1f80f079d43318937e0a0cc151a8546e9e307bef204738b1367f9b29fd`。`local_gravitino_minimum_privilege_verified=true` 只描述本地 Basic rehearsal；OIDC、TLS、持久 catalog 和 production identity 均仍为 `false`。 +- Metadata Fabric M3-7 已建立 production identity profile/attestation gate；checked-in profile fingerprint 为 `2e9d5cac3560b853820f923669f6794ead63bcb36a528639fc0e9539e148ee2f`，report fingerprint 为 `c607589ee25a87acc8a1ab71372618a9a4c10c1e8ebff15b8db7e78b37600b9f`，`profile_valid=true`，40 项 federation/provider/TLS/catalog/tenancy/operations 外部输入以 blockers 暴露，`ready_for_protected_verification=false`、`production_identity_gate_passed=false`、`production_ready=false`。合成完整 attestation 只验证门禁逻辑，不计入生产证据。 ## 下一验收证据 @@ -93,5 +95,5 @@ - staging 的 migration role、应用 login membership、连接池 role/tenant 复位、双租户 API 和 success finalization 运行产物； - DolphinScheduler adapter 的真实 IAM/OIDC、service token provisioning/轮换、provider 最小权限、binding artifact staging 接入、managed outbox worker/provider callback 实际扩容部署、唯一 worker ID、status/lease 故障恢复和无双写证据； - 首条真实图斑链对 golden slice 的 output hash、独立质量结果/evidence、血缘、发布 revision 和 rollback 演练； -- OpenMetadata/Gravitino 的 source host/cluster 外生产 backup account/bucket、KMS/TLS/workload identity、PITR/source-loss recovery、RPO/RTO、OIDC、受保护环境 provider NetworkPolicy/tenant isolation、upgrade/rollback、registry provenance、持续 metrics backend/retention/query、真实 alert delivery/SLO owner/runbook，以及受保护 PolicyDecision/Approval、双 provider 最小权限 ingestion、生产持久 binding、受保护 production OpenLineage receiver、无双写 read-back 和 conformance；M1 fixture、M2 本地 evidence/readiness contracts、M3-1 projection candidate、M3-2 local replay、M3-3 临时 binding ledger、M3-4 loopback delivery 与 M3-5 本地临时 OpenMetadata identity 均不计入生产退出门； +- OpenMetadata/Gravitino 的 source host/cluster 外生产 backup account/bucket、KMS/TLS/workload identity、PITR/source-loss recovery、RPO/RTO、OIDC、受保护环境 provider NetworkPolicy/tenant isolation、upgrade/rollback、registry provenance、持续 metrics backend/retention/query、真实 alert delivery/SLO owner/runbook，以及受保护 PolicyDecision/Approval、双 provider 最小权限 ingestion、生产持久 binding、受保护 production OpenLineage receiver、无双写 read-back 和 conformance；M1 fixture、M2 本地 evidence/readiness contracts、M3-1 projection candidate、M3-2 local replay、M3-3 临时 binding ledger、M3-4 loopback delivery、M3-5/M3-6 本地临时 provider identity 与 M3-7 pending profile/合成 attestation 均不计入生产退出门； - DolphinScheduler/Temporal sandbox 的独立数据库、备份恢复、身份、版本和升级责任证明；DolphinScheduler standalone/H2 不计入此退出门。 diff --git a/scripts/metadata-fabric-identity-gate.sh b/scripts/metadata-fabric-identity-gate.sh new file mode 100755 index 00000000..81482009 --- /dev/null +++ b/scripts/metadata-fabric-identity-gate.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +COMMON_GIT_DIR="$(git -C "$ROOT" rev-parse --path-format=absolute --git-common-dir 2>/dev/null || true)" +SHARED_ROOT="" +if [ -n "$COMMON_GIT_DIR" ]; then + SHARED_ROOT="$(cd "$COMMON_GIT_DIR/.." && pwd)" +fi + +if [ -n "${PYTHON:-}" ]; then + : +elif [ -x "$ROOT/.venv/bin/python" ]; then + PYTHON="$ROOT/.venv/bin/python" +elif [ -n "$SHARED_ROOT" ] && [ -x "$SHARED_ROOT/.venv/bin/python" ]; then + PYTHON="$SHARED_ROOT/.venv/bin/python" +else + PYTHON="python" +fi + +cd "$ROOT" +exec "$PYTHON" -m data_agent.metadata_fabric_identity_gate "$@"