diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0dd3cfd2..bf55ab36 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,8 @@ on: - feat/ar1-metadata-fabric-ingestion-contract - feat/ar1-metadata-fabric-local-ingestion-replay - feat/ar1-metadata-fabric-binding-ledger + - feat/ar1-metadata-fabric-openlineage-delivery + - feat/ar1-metadata-fabric-provider-identity env: PYTHON_VERSION: "3.13" @@ -128,6 +130,9 @@ jobs: - name: Validate metadata fabric OpenLineage delivery evidence run: python -m data_agent.metadata_fabric_lineage_delivery validate + - name: Validate metadata fabric provider identity evidence + run: python -m data_agent.metadata_fabric_provider_identity validate + - name: Validate DolphinScheduler adapter boundary run: python -m data_agent.dolphinscheduler_adapter validate @@ -183,6 +188,7 @@ jobs: data_agent/test_metadata_fabric_ingestion_replay.py \ data_agent/test_metadata_fabric_binding_contract.py \ data_agent/test_metadata_fabric_lineage_delivery.py \ + data_agent/test_metadata_fabric_provider_identity.py \ data_agent/test_metadata_fabric_otel_failure_rehearsal.py \ data_agent/test_metadata_fabric_otel_metrics.py \ data_agent/test_metadata_fabric_provider_metrics.py \ diff --git a/config/metadata-fabric-provider-identity.local.yaml b/config/metadata-fabric-provider-identity.local.yaml new file mode 100644 index 00000000..ee187a61 --- /dev/null +++ b/config/metadata-fabric-provider-identity.local.yaml @@ -0,0 +1,59 @@ +schema: gda.metadata_fabric_provider_identity_profile.v1 +environment: local_docker_desktop + +cluster: + context: docker-desktop + namespace: gda-metadata-sandbox + +providers: + openmetadata: + version: 1.13.1 + workload: deployment/openmetadata + service: openmetadata + service_port: 8585 + service_account: openmetadata + auth_mode: local_basic_bootstrap_provisioner + bootstrap_username_env: GDA_OPENMETADATA_USERNAME + bootstrap_password_env: GDA_OPENMETADATA_PASSWORD + gravitino: + version: 1.3.0 + workload: statefulset/metadata-gravitino + service: metadata-gravitino + service_port: 8090 + authenticator: simple + access_control_enabled: false + +identity: + policy: GdaMetadataTableProjectionPolicy + role: GdaMetadataTableProjectionRole + user: gda-metadata-table-projection + email: gda-metadata-table-projection@open-metadata.org + bot: gda-metadata-table-projection-bot + mandatory_default_role: DefaultBotRole + jwt_expiry: OneHour + allowed_rule: + name: GdaMetadataTableCreate + effect: allow + operations: [Create] + resources: [table] + +probes: + allowed_table_create: + database_schema: gda_lakehouse.land_use.published + table: gda_provider_identity_probe + denied_policy_create: + collection: policies + operation: Create + name: GdaUnauthorizedPolicyProbe + +claims: + local_openmetadata_bounded_identity_verified: false + local_openmetadata_minimum_privilege_verified: false + local_openmetadata_jwt_rotation_verified: false + local_openmetadata_jwt_revocation_verified: false + provider_minimum_privilege_verified: false + protected_workload_identity_verified: false + oidc_verified: false + gravitino_authentication_verified: false + production_identity_verified: false + production_ready: false diff --git a/data_agent/metadata_fabric_provider_identity.py b/data_agent/metadata_fabric_provider_identity.py new file mode 100644 index 00000000..a3185107 --- /dev/null +++ b/data_agent/metadata_fabric_provider_identity.py @@ -0,0 +1,1256 @@ +"""Rehearse bounded OpenMetadata identity and JWT lifecycle locally. + +The bootstrap administrator only provisions and removes an ephemeral provider +identity. The identity itself must create and read one table, be denied policy +creation, survive JWT rotation, and fail after revocation. Gravitino 1.3.0 +remains explicitly outside the verified identity boundary because the sandbox +has its simple authenticator with access control disabled. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import os +import subprocess +import sys +from collections.abc import Mapping +from datetime import UTC, datetime +from pathlib import Path +from typing import Annotated, Any, Literal +from urllib.parse import quote +from uuid import UUID + +import httpx +import yaml +from pydantic import BaseModel, ConfigDict, Field, SecretStr, StringConstraints + +from . import metadata_fabric_ingestion_replay as ingestion_replay +from . import metadata_fabric_provider_metrics as provider_metrics +from . import metadata_fabric_recovery_rehearsal as recovery +from . import metadata_fabric_sandbox as sandbox + + +PROFILE_SCHEMA = "gda.metadata_fabric_provider_identity_profile.v1" +CONTRACT_SCHEMA = "gda.metadata_fabric_provider_identity_contract.v1" +OBSERVATION_SCHEMA = "gda.metadata_fabric_provider_identity_observation.v1" +EVIDENCE_SCHEMA = "gda.metadata_fabric_provider_identity_evidence.v1" +VALIDATION_SCHEMA = "gda.metadata_fabric_provider_identity_validation.v1" +CONTEXT = "docker-desktop" +NAMESPACE = sandbox.NAMESPACE + +REPO_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_PROFILE_PATH = REPO_ROOT / "config/metadata-fabric-provider-identity.local.yaml" +DEFAULT_EVIDENCE_PATH = ( + REPO_ROOT / "docs/evidence/metadata-fabric-provider-identity-2026-07-28.json" +) +DEFAULT_WRAPPER_PATH = REPO_ROOT / "scripts/metadata-fabric-provider-identity.sh" +OPENMETADATA_VALUES_PATH = ( + REPO_ROOT / "helm/metadata-fabric-sandbox/openmetadata-values.yaml" +) +GRAVITINO_MANIFEST_PATH = REPO_ROOT / "k8s/metadata-fabric-sandbox/gravitino.yaml" + +NonEmptyText = Annotated[ + str, + StringConstraints(strip_whitespace=True, min_length=1, max_length=1024), +] + + +class MetadataFabricProviderIdentityError(RuntimeError): + """The bounded provider identity contract or rehearsal failed closed.""" + + +class ProviderRequestError(MetadataFabricProviderIdentityError): + """An allowlisted OpenMetadata request failed without exposing its body.""" + + +class _FrozenModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True) + + +class ClusterProfile(_FrozenModel): + context: Literal["docker-desktop"] + namespace: Literal["gda-metadata-sandbox"] + + +class OpenMetadataProfile(_FrozenModel): + version: Literal["1.13.1"] + workload: Literal["deployment/openmetadata"] + service: Literal["openmetadata"] + service_port: Literal[8585] + service_account: Literal["openmetadata"] + auth_mode: Literal["local_basic_bootstrap_provisioner"] + bootstrap_username_env: Literal["GDA_OPENMETADATA_USERNAME"] + bootstrap_password_env: Literal["GDA_OPENMETADATA_PASSWORD"] + + +class GravitinoProfile(_FrozenModel): + version: Literal["1.3.0"] + workload: Literal["statefulset/metadata-gravitino"] + service: Literal["metadata-gravitino"] + service_port: Literal[8090] + authenticator: Literal["simple"] + access_control_enabled: Literal[False] + + +class ProviderProfiles(_FrozenModel): + openmetadata: OpenMetadataProfile + gravitino: GravitinoProfile + + +class AllowedRule(_FrozenModel): + name: Literal["GdaMetadataTableCreate"] + effect: Literal["allow"] + operations: tuple[Literal["Create"], ...] + resources: tuple[Literal["table"], ...] + + +class IdentityProfile(_FrozenModel): + policy: Literal["GdaMetadataTableProjectionPolicy"] + role: Literal["GdaMetadataTableProjectionRole"] + user: Literal["gda-metadata-table-projection"] + email: Literal["gda-metadata-table-projection@open-metadata.org"] + bot: Literal["gda-metadata-table-projection-bot"] + mandatory_default_role: Literal["DefaultBotRole"] + jwt_expiry: Literal["OneHour"] + allowed_rule: AllowedRule + + +class AllowedTableCreateProbe(_FrozenModel): + database_schema: Literal["gda_lakehouse.land_use.published"] + table: Literal["gda_provider_identity_probe"] + + @property + def fully_qualified_name(self) -> str: + return f"{self.database_schema}.{self.table}" + + +class DeniedPolicyCreateProbe(_FrozenModel): + collection: Literal["policies"] + operation: Literal["Create"] + name: Literal["GdaUnauthorizedPolicyProbe"] + + +class ProbeProfile(_FrozenModel): + allowed_table_create: AllowedTableCreateProbe + denied_policy_create: DeniedPolicyCreateProbe + + +class ClaimProfile(_FrozenModel): + local_openmetadata_bounded_identity_verified: Literal[False] + local_openmetadata_minimum_privilege_verified: Literal[False] + local_openmetadata_jwt_rotation_verified: Literal[False] + local_openmetadata_jwt_revocation_verified: Literal[False] + provider_minimum_privilege_verified: Literal[False] + protected_workload_identity_verified: Literal[False] + oidc_verified: Literal[False] + gravitino_authentication_verified: Literal[False] + production_identity_verified: Literal[False] + production_ready: Literal[False] + + +class ProviderIdentityProfile(_FrozenModel): + schema_name: Literal["gda.metadata_fabric_provider_identity_profile.v1"] = Field( + alias="schema" + ) + environment: Literal["local_docker_desktop"] + cluster: ClusterProfile + providers: ProviderProfiles + identity: IdentityProfile + probes: ProbeProfile + claims: ClaimProfile + + +def _mapping(value: Any) -> Mapping[str, Any]: + return value if isinstance(value, Mapping) else {} + + +def _valid_sha256(value: Any) -> bool: + return ( + isinstance(value, str) + and len(value) == 64 + and all(character in "0123456789abcdef" for character in value) + ) + + +def _valid_uuid(value: Any) -> bool: + try: + UUID(str(value)) + except (TypeError, ValueError): + return False + return True + + +def _load_yaml_object(path: Path) -> dict[str, Any]: + value = yaml.safe_load(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError("provider identity profile must be an object") + return value + + +def load_profile(path: Path = DEFAULT_PROFILE_PATH) -> ProviderIdentityProfile: + try: + value = _load_yaml_object(path.resolve()) + ingestion_replay._reject_sensitive_fields(value) + profile = ProviderIdentityProfile.model_validate(value) + except (OSError, TypeError, ValueError, yaml.YAMLError) as exc: + raise MetadataFabricProviderIdentityError( + "provider identity profile is invalid" + ) from exc + if ( + profile.identity.allowed_rule.operations != ("Create",) + or profile.identity.allowed_rule.resources != ("table",) + ): + raise MetadataFabricProviderIdentityError( + "provider identity policy must allow only table Create" + ) + return profile + + +def build_contract_report( + profile_path: Path = DEFAULT_PROFILE_PATH, + wrapper_path: Path = DEFAULT_WRAPPER_PATH, +) -> dict[str, Any]: + errors: list[str] = [] + profile: ProviderIdentityProfile | None = None + try: + profile = load_profile(profile_path) + except MetadataFabricProviderIdentityError as exc: + errors.append(str(exc)) + + try: + wrapper = wrapper_path.resolve().read_text(encoding="utf-8") + for marker in ("set -euo pipefail", "metadata_fabric_provider_identity"): + if marker not in wrapper: + errors.append(f"provider identity wrapper is missing marker: {marker}") + except OSError as exc: + errors.append(f"provider identity wrapper is invalid: {type(exc).__name__}") + + static_markers = ( + ( + OPENMETADATA_VALUES_PATH, + ( + "provider: basic", + 'initialAdmins: ["admin"]', + "automountServiceAccountToken: false", + ), + ), + ( + GRAVITINO_MANIFEST_PATH, + ( + "gravitino.authenticators = simple", + "gravitino.authorization.enable = false", + ), + ), + ) + for path, markers in static_markers: + try: + content = path.read_text(encoding="utf-8") + for marker in markers: + if marker not in content: + errors.append(f"provider identity source is missing marker: {marker}") + except OSError as exc: + errors.append(f"provider identity source is invalid: {type(exc).__name__}") + + files: dict[str, dict[str, str]] = {} + for path in ( + Path(__file__).resolve(), + profile_path.resolve(), + wrapper_path.resolve(), + OPENMETADATA_VALUES_PATH, + GRAVITINO_MANIFEST_PATH, + ): + if path.is_file(): + try: + relative = path.relative_to(REPO_ROOT).as_posix() + except ValueError: + relative = path.name + files[relative] = { + "path": relative, + "sha256": recovery._file_sha256(path), + } + + identity = profile.identity if profile else None + stable = { + "schema": CONTRACT_SCHEMA, + "context": CONTEXT, + "namespace": NAMESPACE, + "openmetadata_version": sandbox.OPENMETADATA_VERSION, + "gravitino_version": sandbox.GRAVITINO_VERSION, + "identity": ( + { + "policy": identity.policy, + "role": identity.role, + "user": identity.user, + "bot": identity.bot, + "mandatory_default_role": identity.mandatory_default_role, + "allowed_rule": identity.allowed_rule.model_dump(mode="json"), + } + if identity + else None + ), + "local_static_contract_verified": not errors, + "local_openmetadata_bounded_identity_verified": False, + "local_openmetadata_minimum_privilege_verified": False, + "provider_minimum_privilege_verified": False, + "protected_workload_identity_verified": False, + "oidc_verified": False, + "gravitino_authentication_verified": False, + "production_identity_verified": False, + "production_ready": False, + "files": files, + "errors": errors, + } + return {**stable, "contract_fingerprint": recovery._canonical_sha256(stable)} + + +def _json_object(response: httpx.Response, label: str) -> dict[str, Any]: + try: + value = response.json() + except ValueError as exc: + raise ProviderRequestError(f"{label} returned invalid JSON") from exc + if not isinstance(value, dict): + raise ProviderRequestError(f"{label} response is not an object") + return value + + +class _HttpApi: + def __init__( + self, + *, + base_url: str, + bearer: SecretStr, + transport: httpx.BaseTransport | None, + ) -> None: + self._client = httpx.Client( + base_url=base_url.rstrip("/") + "/", + headers={ + "Accept": "application/json", + "Authorization": f"Bearer {bearer.get_secret_value()}", + }, + timeout=30.0, + transport=transport, + ) + + def close(self) -> None: + self._client.close() + + def request( + self, + method: str, + path: str, + *, + json_body: Mapping[str, Any] | None = None, + params: Mapping[str, str] | None = None, + expected: frozenset[int] = frozenset({200}), + label: str, + ) -> tuple[int, dict[str, Any] | None]: + try: + response = self._client.request( + method, + path, + json=dict(json_body) if json_body is not None else None, + params=params, + ) + except httpx.HTTPError as exc: + raise ProviderRequestError(f"{label} request failed") from exc + if response.status_code not in expected: + raise ProviderRequestError( + f"{label} returned unexpected status {response.status_code}" + ) + if response.status_code in {401, 403, 404} or not response.content: + return response.status_code, None + return response.status_code, _json_object(response, label) + + +class OpenMetadataIdentityRehearsal: + """Provision, exercise, revoke, and remove one local provider identity.""" + + def __init__( + self, + *, + base_url: str, + username: str, + password: SecretStr, + transport: httpx.BaseTransport | None = None, + ) -> None: + login = httpx.Client( + base_url=base_url.rstrip("/") + "/", + timeout=30.0, + transport=transport, + ) + encoded_password = base64.b64encode( + password.get_secret_value().encode("utf-8") + ).decode("ascii") + try: + response = login.post( + "users/login", + json={"email": username, "password": encoded_password}, + ) + except httpx.HTTPError as exc: + login.close() + raise ProviderRequestError("OpenMetadata bootstrap login failed") from exc + login.close() + if response.status_code != 200: + raise ProviderRequestError("OpenMetadata bootstrap login was rejected") + payload = _json_object(response, "OpenMetadata bootstrap login") + access = payload.get("accessToken") + if not isinstance(access, str) or not access: + raise ProviderRequestError("OpenMetadata bootstrap login omitted access") + self._base_url = base_url + self._transport = transport + self._admin = _HttpApi( + base_url=base_url, + bearer=SecretStr(access), + transport=transport, + ) + self._created: dict[str, tuple[str, UUID]] = {} + self._bot_clients: list[_HttpApi] = [] + + def close(self) -> None: + for client in self._bot_clients: + client.close() + self._bot_clients.clear() + self._admin.close() + + @staticmethod + def _entity_id(payload: Mapping[str, Any], label: str) -> UUID: + try: + return UUID(str(payload["id"])) + except (KeyError, TypeError, ValueError) as exc: + raise ProviderRequestError(f"{label} omitted provider identity") from exc + + @staticmethod + def _name_path(collection: str, name: str) -> str: + return f"{collection}/name/{quote(name, safe='')}" + + def _lookup( + self, collection: str, name: str, *, fields: str | None = None + ) -> tuple[int, dict[str, Any] | None]: + params = {"fields": fields} if fields else None + return self._admin.request( + "GET", + self._name_path(collection, name), + params=params, + expected=frozenset({200, 404}), + label=f"OpenMetadata {collection} lookup", + ) + + def _preflight(self, profile: ProviderIdentityProfile) -> None: + checks = ( + ("policies", profile.identity.policy), + ("roles", profile.identity.role), + ("users", profile.identity.user), + ("bots", profile.identity.bot), + ("tables", profile.probes.allowed_table_create.fully_qualified_name), + ("policies", profile.probes.denied_policy_create.name), + ) + for collection, name in checks: + status, _payload = self._lookup(collection, name) + if status != 404: + raise MetadataFabricProviderIdentityError( + f"OpenMetadata rehearsal target already exists: {collection}/{name}" + ) + + def _create( + self, + *, + key: str, + collection: str, + payload: Mapping[str, Any], + ) -> dict[str, Any]: + _status, created = self._admin.request( + "PUT", + collection, + json_body=payload, + expected=frozenset({200, 201}), + label=f"OpenMetadata {collection} create", + ) + assert created is not None + self._created[key] = ( + collection, + self._entity_id(created, f"OpenMetadata {collection}"), + ) + return created + + def _bot_api(self, value: str) -> _HttpApi: + client = _HttpApi( + base_url=self._base_url, + bearer=SecretStr(value), + transport=self._transport, + ) + self._bot_clients.append(client) + return client + + def _current_jwt(self, user_id: UUID) -> str: + _status, mechanism = self._admin.request( + "GET", + f"users/auth-mechanism/{user_id}", + label="OpenMetadata bot authentication mechanism", + ) + config = _mapping(_mapping(mechanism).get("config")) + value = config.get("JWTToken") + if not isinstance(value, str) or not value: + raise ProviderRequestError("OpenMetadata bot authentication omitted JWT") + return value + + def _cleanup(self, profile: ProviderIdentityProfile) -> dict[str, Any]: + revoke_attempted = False + user = self._created.get("user") + if user is not None: + revoke_attempted = True + try: + self._admin.request( + "PUT", + "users/revokeToken", + json_body={"id": str(user[1])}, + expected=frozenset({200, 400, 404}), + label="OpenMetadata bot JWT cleanup revocation", + ) + except ProviderRequestError: + pass + + delete_order = ("table", "unauthorized_policy", "bot", "user", "role", "policy") + delete_statuses: dict[str, int] = {} + for key in delete_order: + created = self._created.get(key) + if created is None: + continue + collection, entity_id = created + try: + status, _payload = self._admin.request( + "DELETE", + f"{collection}/{entity_id}", + params={"recursive": "true", "hardDelete": "true"}, + expected=frozenset({200, 204, 404}), + label=f"OpenMetadata {key} cleanup", + ) + delete_statuses[key] = status + except ProviderRequestError: + delete_statuses[key] = 0 + + absence_checks = ( + ("table", "tables", profile.probes.allowed_table_create.fully_qualified_name), + ("bot", "bots", profile.identity.bot), + ("user", "users", profile.identity.user), + ("role", "roles", profile.identity.role), + ("policy", "policies", profile.identity.policy), + ( + "unauthorized_policy", + "policies", + profile.probes.denied_policy_create.name, + ), + ) + absent: dict[str, bool] = {} + for key, collection, name in absence_checks: + try: + status, _payload = self._lookup(collection, name) + absent[key] = status == 404 + except ProviderRequestError: + absent[key] = False + return { + "jwt_revocation_attempted": revoke_attempted, + "delete_statuses": delete_statuses, + "all_rehearsal_objects_absent": all(absent.values()), + "absence": absent, + } + + def execute(self, profile: ProviderIdentityProfile) -> dict[str, Any]: + self._preflight(profile) + outcome: dict[str, Any] | None = None + try: + rule = profile.identity.allowed_rule + policy = self._create( + key="policy", + collection="policies", + payload={ + "name": profile.identity.policy, + "description": "Bounded local GDA table projection rehearsal", + "enabled": True, + "rules": [rule.model_dump(mode="json")], + }, + ) + policy_id = self._entity_id(policy, "OpenMetadata policy") + role = self._create( + key="role", + collection="roles", + payload={ + "name": profile.identity.role, + "description": "Bounded local GDA table projection role", + "policies": [profile.identity.policy], + }, + ) + role_id = self._entity_id(role, "OpenMetadata role") + user = self._create( + key="user", + collection="users", + payload={ + "name": profile.identity.user, + "email": profile.identity.email, + "description": "Bounded local GDA metadata table projection identity", + "isBot": True, + "isAdmin": False, + "roles": [str(role_id)], + "authenticationMechanism": { + "authType": "JWT", + "config": {"JWTTokenExpiry": profile.identity.jwt_expiry}, + }, + }, + ) + user_id = self._entity_id(user, "OpenMetadata user") + bot = self._create( + key="bot", + collection="bots", + payload={ + "name": profile.identity.bot, + "description": "Bounded local GDA metadata table projection bot", + "botUser": profile.identity.user, + "provider": "automation", + }, + ) + bot_id = self._entity_id(bot, "OpenMetadata bot") + + initial_jwt = self._current_jwt(user_id) + initial_api = self._bot_api(initial_jwt) + principal_status, principal = initial_api.request( + "GET", + "users/loggedInUser", + params={"fields": "roles"}, + label="OpenMetadata bot principal", + ) + assert principal is not None + role_names = sorted( + str(item.get("name")) + for item in principal.get("roles", []) + if isinstance(item, Mapping) + ) + + table_probe = profile.probes.allowed_table_create + table_status, table = initial_api.request( + "PUT", + "tables", + json_body={ + "name": table_probe.table, + "description": "Ephemeral local minimum privilege probe", + "databaseSchema": table_probe.database_schema, + "tableType": "Regular", + "columns": [ + { + "name": "probe_id", + "dataType": "STRING", + "constraint": "NOT_NULL", + } + ], + }, + expected=frozenset({200, 201}), + label="OpenMetadata allowed table create probe", + ) + assert table is not None + table_id = self._entity_id(table, "OpenMetadata probe table") + self._created["table"] = ("tables", table_id) + read_status, readback = initial_api.request( + "GET", + self._name_path("tables", table_probe.fully_qualified_name), + label="OpenMetadata allowed table readback probe", + ) + assert readback is not None + + denied = profile.probes.denied_policy_create + denied_status, denied_payload = initial_api.request( + "PUT", + denied.collection, + json_body={ + "name": denied.name, + "enabled": True, + "rules": [ + { + "name": "MustNotCreate", + "effect": "allow", + "operations": ["All"], + "resources": ["All"], + } + ], + }, + expected=frozenset({200, 201, 403}), + label="OpenMetadata denied policy create probe", + ) + if denied_status in {200, 201} and denied_payload is not None: + self._created["unauthorized_policy"] = ( + "policies", + self._entity_id( + denied_payload, "OpenMetadata unauthorized policy probe" + ), + ) + + _rotation_status, rotated = self._admin.request( + "PUT", + f"users/generateToken/{user_id}", + json_body={"JWTTokenExpiry": profile.identity.jwt_expiry}, + label="OpenMetadata bot JWT rotation", + ) + rotated_jwt = _mapping(rotated).get("JWTToken") + if not isinstance(rotated_jwt, str) or not rotated_jwt: + raise ProviderRequestError("OpenMetadata rotation omitted JWT") + old_after_rotation, _payload = initial_api.request( + "GET", + "users/loggedInUser", + expected=frozenset({200, 401, 403}), + label="OpenMetadata old JWT after rotation", + ) + rotated_api = self._bot_api(rotated_jwt) + new_after_rotation, rotated_principal = rotated_api.request( + "GET", + "users/loggedInUser", + params={"fields": "roles"}, + expected=frozenset({200, 401, 403}), + label="OpenMetadata rotated JWT", + ) + _revoke_status, _revoked = self._admin.request( + "PUT", + "users/revokeToken", + json_body={"id": str(user_id)}, + label="OpenMetadata bot JWT revocation", + ) + new_after_revocation, _payload = rotated_api.request( + "GET", + "users/loggedInUser", + expected=frozenset({200, 401, 403}), + label="OpenMetadata revoked JWT", + ) + + outcome = { + "provisioner": { + "auth_mode": profile.providers.openmetadata.auth_mode, + "bootstrap_admin_used": True, + "minimum_privilege": False, + }, + "principal": { + "id": str(user_id), + "name": principal.get("name"), + "is_admin": principal.get("isAdmin"), + "is_bot": principal.get("isBot"), + "effective_roles": role_names, + "provider_mandatory_default_role_inherited": True, + "minimum_privilege_scope": ( + "dedicated_table_create_grant_with_provider_mandatory_default_role" + ), + }, + "policy": { + "id": str(policy_id), + "name": policy.get("name"), + "enabled": policy.get("enabled"), + "rules": policy.get("rules"), + }, + "role": { + "id": str(role_id), + "name": role.get("name"), + "policy_ids": [str(policy_id)], + "mandatory_default_role": profile.identity.mandatory_default_role, + }, + "bot": { + "id": str(bot_id), + "name": bot.get("name"), + "provider": bot.get("provider"), + }, + "allowed_probe": { + "operation": "Create", + "resource": "table", + "create_status": table_status, + "read_status": read_status, + "entity_id": str(table_id), + "fully_qualified_name": readback.get("fullyQualifiedName"), + }, + "denied_probe": { + "operation": denied.operation, + "resource": "policy", + "status": denied_status, + }, + "jwt_lifecycle": { + "expiry": profile.identity.jwt_expiry, + "initial_authentication_status": principal_status, + "old_after_rotation_status": old_after_rotation, + "rotated_authentication_status": new_after_rotation, + "rotated_principal_matches": ( + _mapping(rotated_principal).get("id") == str(user_id) + ), + "after_revocation_status": new_after_revocation, + "sensitive_material_recorded": False, + }, + } + except BaseException: + self._cleanup(profile) + raise + + assert outcome is not None + outcome["cleanup"] = self._cleanup(profile) + return outcome + + +def _kubectl_json(args: list[str]) -> dict[str, Any]: + try: + completed = subprocess.run( + ["kubectl", "--context", CONTEXT, *args], + capture_output=True, + check=False, + text=True, + timeout=60, + ) + except (OSError, subprocess.SubprocessError) as exc: + raise MetadataFabricProviderIdentityError("kubectl is unavailable") from exc + if completed.returncode != 0: + raise MetadataFabricProviderIdentityError("kubectl identity observation failed") + try: + value = json.loads(completed.stdout) + except json.JSONDecodeError as exc: + raise MetadataFabricProviderIdentityError( + "kubectl identity observation is invalid JSON" + ) from exc + if not isinstance(value, dict): + raise MetadataFabricProviderIdentityError( + "kubectl identity observation is not an object" + ) + return value + + +def _runtime_identity(profile: ProviderIdentityProfile) -> dict[str, Any]: + namespace = _kubectl_json(["get", "namespace", NAMESPACE, "-o", "json"]) + service = _kubectl_json( + ["-n", NAMESPACE, "get", "service", "openmetadata", "-o", "json"] + ) + deployment = _kubectl_json( + ["-n", NAMESPACE, "get", "deployment", "openmetadata", "-o", "json"] + ) + pod_spec = _mapping( + _mapping(_mapping(deployment.get("spec")).get("template")).get("spec") + ) + containers = ( + pod_spec.get("containers") + if isinstance(pod_spec.get("containers"), list) + else [] + ) + container = next( + ( + _mapping(item) + for item in containers + if _mapping(item).get("name") == "openmetadata" + ), + {}, + ) + return { + "context": profile.cluster.context, + "namespace": { + "name": _mapping(namespace.get("metadata")).get("name"), + "uid": _mapping(namespace.get("metadata")).get("uid"), + }, + "service": { + "name": _mapping(service.get("metadata")).get("name"), + "uid": _mapping(service.get("metadata")).get("uid"), + "type": _mapping(service.get("spec")).get("type"), + }, + "workload": { + "kind": deployment.get("kind"), + "name": _mapping(deployment.get("metadata")).get("name"), + "uid": _mapping(deployment.get("metadata")).get("uid"), + "image": container.get("image"), + "service_account": pod_spec.get("serviceAccountName"), + "service_account_automount_disabled": ( + pod_spec.get("automountServiceAccountToken") is False + ), + "ready_replicas": _mapping(deployment.get("status")).get( + "readyReplicas", 0 + ), + }, + } + + +def build_evidence(observation: Mapping[str, Any]) -> dict[str, Any]: + errors: list[str] = [] + try: + ingestion_replay._reject_sensitive_fields(observation) + except ValueError: + errors.append("provider identity observation contains sensitive material") + if observation.get("schema") != OBSERVATION_SCHEMA: + errors.append("provider identity observation schema does not match") + + contract = _mapping(observation.get("contract")) + if ( + contract.get("local_static_contract_verified") is not True + or not _valid_sha256(contract.get("contract_fingerprint")) + ): + errors.append("provider identity static contract is not bound") + + runtime = _mapping(observation.get("runtime")) + workload = _mapping(runtime.get("workload")) + if ( + runtime.get("context") != CONTEXT + or _mapping(runtime.get("namespace")).get("name") != NAMESPACE + or _mapping(runtime.get("service")).get("name") != "openmetadata" + or _mapping(runtime.get("service")).get("type") != "ClusterIP" + or workload.get("kind") != "Deployment" + or workload.get("name") != "openmetadata" + or workload.get("image") + != "docker.getcollate.io/openmetadata/server:1.13.1" + or workload.get("service_account") != "openmetadata" + or workload.get("service_account_automount_disabled") is not True + or workload.get("ready_replicas") != 1 + ): + errors.append("OpenMetadata runtime identity does not match the sandbox") + + provider = _mapping(observation.get("openmetadata")) + provisioner = _mapping(provider.get("provisioner")) + if ( + provisioner.get("auth_mode") != "local_basic_bootstrap_provisioner" + or provisioner.get("bootstrap_admin_used") is not True + or provisioner.get("minimum_privilege") is not False + ): + errors.append("OpenMetadata bootstrap provisioner boundary is missing") + + principal = _mapping(provider.get("principal")) + expected_roles = ["DefaultBotRole", "GdaMetadataTableProjectionRole"] + if ( + not _valid_uuid(principal.get("id")) + or + principal.get("name") != "gda-metadata-table-projection" + or principal.get("is_admin") is not False + or principal.get("is_bot") is not True + or principal.get("effective_roles") != expected_roles + or principal.get("provider_mandatory_default_role_inherited") is not True + or principal.get("minimum_privilege_scope") + != "dedicated_table_create_grant_with_provider_mandatory_default_role" + ): + errors.append("OpenMetadata principal or effective roles do not match") + + policy = _mapping(provider.get("policy")) + expected_rule = { + "name": "GdaMetadataTableCreate", + "effect": "allow", + "operations": ["Create"], + "resources": ["table"], + } + observed_rules = policy.get("rules") + normalized_rules = [] + if isinstance(observed_rules, list): + for item in observed_rules: + rule = _mapping(item) + normalized_rules.append( + { + "name": rule.get("name"), + "effect": rule.get("effect"), + "operations": rule.get("operations"), + "resources": rule.get("resources"), + } + ) + if ( + not _valid_uuid(policy.get("id")) + or policy.get("name") != "GdaMetadataTableProjectionPolicy" + or policy.get("enabled") is not True + or normalized_rules != [expected_rule] + ): + errors.append("OpenMetadata dedicated policy is broader than table Create") + + role = _mapping(provider.get("role")) + if ( + not _valid_uuid(role.get("id")) + or role.get("name") != "GdaMetadataTableProjectionRole" + or role.get("policy_ids") != [policy.get("id")] + or role.get("mandatory_default_role") != "DefaultBotRole" + ): + errors.append("OpenMetadata role does not bind only the dedicated policy") + + bot = _mapping(provider.get("bot")) + if ( + not _valid_uuid(bot.get("id")) + or bot.get("name") != "gda-metadata-table-projection-bot" + or bot.get("provider") != "automation" + ): + errors.append("OpenMetadata bot identity does not match") + + allowed = _mapping(provider.get("allowed_probe")) + if ( + allowed.get("operation") != "Create" + or allowed.get("resource") != "table" + or not _valid_uuid(allowed.get("entity_id")) + or allowed.get("create_status") not in {200, 201} + or allowed.get("read_status") != 200 + or allowed.get("fully_qualified_name") + != "gda_lakehouse.land_use.published.gda_provider_identity_probe" + ): + errors.append("OpenMetadata allowed table probe did not pass") + + denied = _mapping(provider.get("denied_probe")) + if ( + denied.get("operation") != "Create" + or denied.get("resource") != "policy" + or denied.get("status") != 403 + ): + errors.append("OpenMetadata policy-create denial was not enforced") + + lifecycle = _mapping(provider.get("jwt_lifecycle")) + if ( + lifecycle.get("expiry") != "OneHour" + or lifecycle.get("initial_authentication_status") != 200 + or lifecycle.get("old_after_rotation_status") != 401 + or lifecycle.get("rotated_authentication_status") != 200 + or lifecycle.get("rotated_principal_matches") is not True + or lifecycle.get("after_revocation_status") != 401 + or lifecycle.get("sensitive_material_recorded") is not False + ): + errors.append("OpenMetadata JWT rotation or revocation did not pass") + + cleanup = _mapping(provider.get("cleanup")) + runtime_checks = _mapping(observation.get("runtime_checks")) + absence = _mapping(cleanup.get("absence")) + delete_statuses = _mapping(cleanup.get("delete_statuses")) + if ( + cleanup.get("all_rehearsal_objects_absent") is not True + or absence + != { + "table": True, + "bot": True, + "user": True, + "role": True, + "policy": True, + "unauthorized_policy": True, + } + or set(delete_statuses) != {"table", "bot", "user", "role", "policy"} + or any(status not in {200, 204, 404} for status in delete_statuses.values()) + or runtime_checks.get("all_port_forwards_stopped") is not True + or runtime_checks.get("provider_objects_retained") is not False + or runtime_checks.get("sensitive_material_recorded") is not False + or runtime_checks.get("kubernetes_service_account_used_for_provider_login") + is not False + ): + errors.append("provider identity rehearsal cleanup is incomplete") + + gravitino = _mapping(observation.get("gravitino")) + if gravitino != { + "version": "1.3.0", + "authenticator": "simple", + "access_control_enabled": False, + "authentication_verified": False, + }: + errors.append("Gravitino blocked identity boundary does not match") + + verified = not errors + stable = { + "schema": EVIDENCE_SCHEMA, + "status": ( + "local_openmetadata_bounded_identity_verified" if verified else "blocked" + ), + "observation": dict(observation), + "errors": errors, + "local_openmetadata_bounded_identity_verified": verified, + "local_openmetadata_minimum_privilege_verified": verified, + "local_openmetadata_jwt_rotation_verified": verified, + "local_openmetadata_jwt_revocation_verified": verified, + "provider_minimum_privilege_verified": False, + "protected_workload_identity_verified": False, + "oidc_verified": False, + "gravitino_authentication_verified": False, + "production_identity_verified": False, + "production_ready": False, + } + return {**stable, "evidence_fingerprint": recovery._canonical_sha256(stable)} + + +def verify_evidence_integrity(evidence: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + if evidence.get("schema") != EVIDENCE_SCHEMA: + errors.append("provider identity evidence schema does not match") + expected = build_evidence(_mapping(evidence.get("observation"))) + if dict(evidence) != expected: + errors.append("provider identity evidence content or fingerprint drifted") + for claim in ( + "provider_minimum_privilege_verified", + "protected_workload_identity_verified", + "oidc_verified", + "gravitino_authentication_verified", + "production_identity_verified", + "production_ready", + ): + if evidence.get(claim) is not False: + errors.append(f"provider identity evidence may not claim {claim}") + if evidence.get("local_openmetadata_bounded_identity_verified") is not True: + errors.append("local OpenMetadata bounded identity is not verified") + return errors + + +def run_live_rehearsal( + profile_path: Path = DEFAULT_PROFILE_PATH, +) -> dict[str, Any]: + profile = load_profile(profile_path) + contract = build_contract_report(profile_path) + if contract.get("local_static_contract_verified") is not True: + raise MetadataFabricProviderIdentityError( + "provider identity static contract is invalid" + ) + runtime = _runtime_identity(profile) + forward = provider_metrics._PortForward( + kubectl="kubectl", + context=profile.cluster.context, + namespace=profile.cluster.namespace, + service=profile.providers.openmetadata.service, + target_port=profile.providers.openmetadata.service_port, + ) + rehearsal: OpenMetadataIdentityRehearsal | None = None + result: dict[str, Any] | None = None + stopped = False + try: + forward.start() + try: + username = os.environ[ + profile.providers.openmetadata.bootstrap_username_env + ] + password = SecretStr( + os.environ[profile.providers.openmetadata.bootstrap_password_env] + ) + except KeyError as exc: + raise MetadataFabricProviderIdentityError( + "OpenMetadata local bootstrap environment is missing" + ) from exc + rehearsal = OpenMetadataIdentityRehearsal( + base_url=f"http://127.0.0.1:{forward.local_port}/api/v1", + username=username, + password=password, + ) + result = rehearsal.execute(profile) + finally: + if rehearsal is not None: + rehearsal.close() + stopped = forward.stop() + if result is None: + raise MetadataFabricProviderIdentityError( + "provider identity rehearsal did not produce an outcome" + ) + observation = { + "schema": OBSERVATION_SCHEMA, + "observed_at": datetime.now(UTC).isoformat(), + "contract": { + "contract_fingerprint": contract["contract_fingerprint"], + "local_static_contract_verified": True, + }, + "runtime": runtime, + "openmetadata": result, + "gravitino": { + "version": profile.providers.gravitino.version, + "authenticator": profile.providers.gravitino.authenticator, + "access_control_enabled": ( + profile.providers.gravitino.access_control_enabled + ), + "authentication_verified": False, + }, + "runtime_checks": { + "all_port_forwards_stopped": stopped, + "provider_objects_retained": False, + "sensitive_material_recorded": False, + "kubernetes_service_account_used_for_provider_login": False, + }, + } + return build_evidence(observation) + + +def build_validation_report( + *, + profile_path: Path = DEFAULT_PROFILE_PATH, + evidence_path: Path = DEFAULT_EVIDENCE_PATH, +) -> dict[str, Any]: + contract = build_contract_report(profile_path) + errors = list(contract["errors"]) + evidence: dict[str, Any] | None = None + try: + value = json.loads(evidence_path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError("provider identity evidence must be an object") + evidence = value + errors.extend(verify_evidence_integrity(evidence)) + observed_contract = _mapping( + _mapping(evidence.get("observation")).get("contract") + ).get("contract_fingerprint") + if observed_contract != contract.get("contract_fingerprint"): + errors.append("provider identity evidence contract fingerprint drift") + except (OSError, TypeError, ValueError, json.JSONDecodeError) as exc: + errors.append(f"provider identity evidence is invalid: {type(exc).__name__}") + verified = not errors + return { + "schema": VALIDATION_SCHEMA, + "local_static_contract_verified": contract[ + "local_static_contract_verified" + ], + "local_openmetadata_bounded_identity_verified": ( + verified + and evidence is not None + and evidence.get("local_openmetadata_bounded_identity_verified") is True + ), + "local_openmetadata_minimum_privilege_verified": ( + verified + and evidence is not None + and evidence.get("local_openmetadata_minimum_privilege_verified") is True + ), + "provider_minimum_privilege_verified": False, + "protected_workload_identity_verified": False, + "gravitino_authentication_verified": False, + "production_identity_verified": False, + "production_ready": False, + "contract_fingerprint": contract["contract_fingerprint"], + "evidence_fingerprint": ( + evidence.get("evidence_fingerprint") if evidence else None + ), + "errors": errors, + } + + +def _write_json(path: Path, payload: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(payload, ensure_ascii=True, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + validate = subparsers.add_parser("validate") + validate.add_argument("--profile", type=Path, default=DEFAULT_PROFILE_PATH) + validate.add_argument("--evidence", type=Path, default=DEFAULT_EVIDENCE_PATH) + rehearse = subparsers.add_parser("rehearse") + rehearse.add_argument("--profile", type=Path, default=DEFAULT_PROFILE_PATH) + rehearse.add_argument("--evidence-out", type=Path, required=True) + verify = subparsers.add_parser("verify") + verify.add_argument("--evidence", type=Path, default=DEFAULT_EVIDENCE_PATH) + args = parser.parse_args(argv) + try: + if args.command == "validate": + report = build_validation_report( + profile_path=args.profile, + evidence_path=args.evidence, + ) + print(json.dumps(report, ensure_ascii=True, indent=2, sort_keys=True)) + return 0 if not report["errors"] else 1 + if args.command == "verify": + value = json.loads(args.evidence.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError("provider identity evidence must be an object") + errors = verify_evidence_integrity(value) + print(json.dumps({"verified": not errors, "errors": errors}, indent=2)) + return 0 if not errors else 1 + evidence = run_live_rehearsal(args.profile) + _write_json(args.evidence_out, evidence) + print(json.dumps(evidence, ensure_ascii=True, indent=2, sort_keys=True)) + return 0 if not evidence["errors"] else 1 + except ( + KeyError, + OSError, + TypeError, + ValueError, + json.JSONDecodeError, + MetadataFabricProviderIdentityError, + KeyboardInterrupt, + ) as exc: + print(f"metadata fabric provider identity: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/data_agent/platform_truth.py b/data_agent/platform_truth.py index 54ac20fb..590a1819 100644 --- a/data_agent/platform_truth.py +++ b/data_agent/platform_truth.py @@ -608,6 +608,26 @@ def _config( ), "Managed outbox worker and protected authenticated OpenLineage receiver", ), + RuntimeSpec( + "metadata_provider_identity_rehearsal", + "provider_identity_rehearsal", + "governed", + "evidence_durable", + "committed local OpenMetadata bounded identity evidence", + "metadata-platform", + "local_verification_only", + ( + "data_agent/metadata_fabric_provider_identity.py", + "scripts/metadata-fabric-provider-identity.sh", + ), + ( + ( + "data_agent/metadata_fabric_provider_identity.py", + "subprocess.run", + ), + ), + "Protected OIDC workload identity and authenticated Gravitino access control", + ), RuntimeSpec( "datalake_monitor", "monitor_loop", diff --git a/data_agent/test_metadata_fabric_provider_identity.py b/data_agent/test_metadata_fabric_provider_identity.py new file mode 100644 index 00000000..d14502fe --- /dev/null +++ b/data_agent/test_metadata_fabric_provider_identity.py @@ -0,0 +1,400 @@ +import json +from copy import deepcopy +from pathlib import Path +from urllib.parse import unquote +from uuid import UUID + +import httpx +import yaml +from pydantic import SecretStr + +from data_agent import metadata_fabric_provider_identity as identity + + +POLICY_ID = UUID("11111111-1111-4111-8111-111111111111") +ROLE_ID = UUID("22222222-2222-4222-8222-222222222222") +USER_ID = UUID("33333333-3333-4333-8333-333333333333") +BOT_ID = UUID("44444444-4444-4444-8444-444444444444") +TABLE_ID = UUID("55555555-5555-4555-8555-555555555555") + + +def _provider_result() -> dict: + return { + "provisioner": { + "auth_mode": "local_basic_bootstrap_provisioner", + "bootstrap_admin_used": True, + "minimum_privilege": False, + }, + "principal": { + "id": str(USER_ID), + "name": "gda-metadata-table-projection", + "is_admin": False, + "is_bot": True, + "effective_roles": [ + "DefaultBotRole", + "GdaMetadataTableProjectionRole", + ], + "provider_mandatory_default_role_inherited": True, + "minimum_privilege_scope": ( + "dedicated_table_create_grant_with_provider_mandatory_default_role" + ), + }, + "policy": { + "id": str(POLICY_ID), + "name": "GdaMetadataTableProjectionPolicy", + "enabled": True, + "rules": [ + { + "name": "GdaMetadataTableCreate", + "description": "bounded", + "effect": "allow", + "operations": ["Create"], + "resources": ["table"], + } + ], + }, + "role": { + "id": str(ROLE_ID), + "name": "GdaMetadataTableProjectionRole", + "policy_ids": [str(POLICY_ID)], + "mandatory_default_role": "DefaultBotRole", + }, + "bot": { + "id": str(BOT_ID), + "name": "gda-metadata-table-projection-bot", + "provider": "automation", + }, + "allowed_probe": { + "operation": "Create", + "resource": "table", + "create_status": 201, + "read_status": 200, + "entity_id": str(TABLE_ID), + "fully_qualified_name": ( + "gda_lakehouse.land_use.published.gda_provider_identity_probe" + ), + }, + "denied_probe": { + "operation": "Create", + "resource": "policy", + "status": 403, + }, + "jwt_lifecycle": { + "expiry": "OneHour", + "initial_authentication_status": 200, + "old_after_rotation_status": 401, + "rotated_authentication_status": 200, + "rotated_principal_matches": True, + "after_revocation_status": 401, + "sensitive_material_recorded": False, + }, + "cleanup": { + "jwt_revocation_attempted": True, + "delete_statuses": { + "table": 200, + "bot": 200, + "user": 404, + "role": 200, + "policy": 200, + }, + "all_rehearsal_objects_absent": True, + "absence": { + "table": True, + "bot": True, + "user": True, + "role": True, + "policy": True, + "unauthorized_policy": True, + }, + }, + } + + +def _observation() -> dict: + return { + "schema": identity.OBSERVATION_SCHEMA, + "observed_at": "2026-07-28T12:00:00+00:00", + "contract": { + "contract_fingerprint": "a" * 64, + "local_static_contract_verified": True, + }, + "runtime": { + "context": identity.CONTEXT, + "namespace": {"name": identity.NAMESPACE, "uid": "namespace-uid"}, + "service": { + "name": "openmetadata", + "uid": "service-uid", + "type": "ClusterIP", + }, + "workload": { + "kind": "Deployment", + "name": "openmetadata", + "uid": "workload-uid", + "image": "docker.getcollate.io/openmetadata/server:1.13.1", + "service_account": "openmetadata", + "service_account_automount_disabled": True, + "ready_replicas": 1, + }, + }, + "openmetadata": _provider_result(), + "gravitino": { + "version": "1.3.0", + "authenticator": "simple", + "access_control_enabled": False, + "authentication_verified": False, + }, + "runtime_checks": { + "all_port_forwards_stopped": True, + "provider_objects_retained": False, + "sensitive_material_recorded": False, + "kubernetes_service_account_used_for_provider_login": False, + }, + } + + +def test_static_provider_identity_contract_is_valid_and_explicitly_local(): + report = identity.build_contract_report() + + assert report["local_static_contract_verified"] is True + assert report["errors"] == [] + assert report["local_openmetadata_bounded_identity_verified"] is False + assert report["provider_minimum_privilege_verified"] is False + assert report["protected_workload_identity_verified"] is False + assert report["gravitino_authentication_verified"] is False + assert report["production_ready"] is False + assert all( + not Path(item["path"]).is_absolute() for item in report["files"].values() + ) + + +def test_profile_rejects_broad_policy_and_claim_overreach(tmp_path): + profile = yaml.safe_load(identity.DEFAULT_PROFILE_PATH.read_text(encoding="utf-8")) + profile["identity"]["allowed_rule"]["operations"] = ["Create", "EditAll"] + profile["identity"]["allowed_rule"]["resources"] = ["All"] + profile["claims"]["production_ready"] = True + target = tmp_path / "profile.yaml" + target.write_text(yaml.safe_dump(profile), encoding="utf-8") + + report = identity.build_contract_report(profile_path=target) + + assert report["local_static_contract_verified"] is False + assert "profile is invalid" in "\n".join(report["errors"]) + + +def test_profile_rejects_inline_sensitive_material(tmp_path): + profile = yaml.safe_load(identity.DEFAULT_PROFILE_PATH.read_text(encoding="utf-8")) + profile["providers"]["openmetadata"]["password"] = "must-not-enter-config" + target = tmp_path / "profile.yaml" + target.write_text(yaml.safe_dump(profile), encoding="utf-8") + + report = identity.build_contract_report(profile_path=target) + + assert report["local_static_contract_verified"] is False + + +def test_valid_evidence_proves_only_bounded_local_openmetadata_identity(): + evidence = identity.build_evidence(_observation()) + + assert evidence["status"] == "local_openmetadata_bounded_identity_verified" + assert evidence["local_openmetadata_bounded_identity_verified"] is True + assert evidence["local_openmetadata_minimum_privilege_verified"] is True + assert evidence["local_openmetadata_jwt_rotation_verified"] is True + assert evidence["local_openmetadata_jwt_revocation_verified"] is True + assert evidence["provider_minimum_privilege_verified"] is False + assert evidence["protected_workload_identity_verified"] is False + assert evidence["gravitino_authentication_verified"] is False + assert evidence["production_ready"] is False + assert identity.verify_evidence_integrity(evidence) == [] + + +def test_evidence_blocks_broad_role_failed_denial_and_incomplete_cleanup(): + observation = _observation() + observation["openmetadata"]["principal"]["effective_roles"].append( + "IngestionBotRole" + ) + observation["openmetadata"]["policy"]["rules"][0]["operations"].append( + "EditAll" + ) + observation["openmetadata"]["denied_probe"]["status"] = 200 + observation["openmetadata"]["cleanup"][ + "all_rehearsal_objects_absent" + ] = False + + evidence = identity.build_evidence(observation) + + assert evidence["status"] == "blocked" + rendered = "\n".join(evidence["errors"]) + assert "principal or effective roles" in rendered + assert "broader than table Create" in rendered + assert "policy-create denial" in rendered + assert "cleanup is incomplete" in rendered + + +def test_evidence_blocks_jwt_lifecycle_drift_and_sensitive_fields(): + observation = _observation() + observation["openmetadata"]["jwt_lifecycle"][ + "old_after_rotation_status" + ] = 200 + observation["openmetadata"]["api_token"] = "must-not-enter-evidence" + + evidence = identity.build_evidence(observation) + + assert evidence["status"] == "blocked" + rendered = "\n".join(evidence["errors"]) + assert "sensitive material" in rendered + assert "rotation or revocation" in rendered + + +def test_evidence_integrity_rejects_claim_and_observation_tampering(): + evidence = identity.build_evidence(_observation()) + tampered = deepcopy(evidence) + tampered["provider_minimum_privilege_verified"] = True + tampered["observation"]["openmetadata"]["denied_probe"]["status"] = 200 + tampered["observation"]["openmetadata"]["role"]["policy_ids"] = [ + str(BOT_ID) + ] + + errors = identity.verify_evidence_integrity(tampered) + + assert any("content or fingerprint drifted" in error for error in errors) + assert any("provider_minimum_privilege_verified" in error for error in errors) + + +class _OpenMetadataMock: + def __init__(self) -> None: + self.current_jwt = "local-jwt-one" + self.revoked = False + self.entities: dict[str, dict[str, dict]] = { + name: {} for name in ("policies", "roles", "users", "bots", "tables") + } + self.ids = { + "policies": POLICY_ID, + "roles": ROLE_ID, + "users": USER_ID, + "bots": BOT_ID, + "tables": TABLE_ID, + } + + @staticmethod + def _response(status: int, payload: dict | None = None) -> httpx.Response: + return httpx.Response(status, json=payload) if payload is not None else httpx.Response(status) + + def __call__(self, request: httpx.Request) -> httpx.Response: + path = request.url.path.removeprefix("/api/v1/") + method = request.method + auth = request.headers.get("Authorization", "") + body = json.loads(request.content) if request.content else {} + + if path == "users/login" and method == "POST": + return self._response(200, {"accessToken": "bootstrap-access"}) + is_admin = auth == "Bearer bootstrap-access" + is_bot = auth == f"Bearer {self.current_jwt}" and not self.revoked + + if path == "users/loggedInUser" and method == "GET": + if not is_bot: + return self._response(401, {"code": 401}) + return self._response( + 200, + { + "id": str(USER_ID), + "name": "gda-metadata-table-projection", + "isAdmin": False, + "isBot": True, + "roles": [ + {"id": "default", "name": "DefaultBotRole"}, + {"id": str(ROLE_ID), "name": "GdaMetadataTableProjectionRole"}, + ], + }, + ) + if path == f"users/auth-mechanism/{USER_ID}" and method == "GET" and is_admin: + return self._response( + 200, + { + "authType": "JWT", + "config": { + "JWTToken": self.current_jwt, + "JWTTokenExpiry": "OneHour", + }, + }, + ) + if path == f"users/generateToken/{USER_ID}" and method == "PUT" and is_admin: + self.current_jwt = "local-jwt-two" + self.revoked = False + return self._response( + 200, + {"JWTToken": self.current_jwt, "JWTTokenExpiry": "OneHour"}, + ) + if path == "users/revokeToken" and method == "PUT" and is_admin: + self.revoked = True + return self._response(200, {"JWTTokenExpiry": "OneHour"}) + + if "/name/" in path and method == "GET" and (is_admin or is_bot): + collection, encoded_name = path.split("/name/", 1) + entity = self.entities.get(collection, {}).get(unquote(encoded_name)) + return self._response(200, entity) if entity else self._response(404, {"code": 404}) + + if path == "policies" and method == "PUT" and is_bot: + return self._response(403, {"code": 403}) + if path == "tables" and method == "PUT" and is_bot: + entity = { + "id": str(TABLE_ID), + "name": body["name"], + "fullyQualifiedName": ( + f"{body['databaseSchema']}.{body['name']}" + ), + "version": 0.1, + } + self.entities["tables"][entity["fullyQualifiedName"]] = entity + return self._response(201, entity) + + if method == "PUT" and path in self.entities and is_admin: + entity_id = self.ids[path] + entity = {**body, "id": str(entity_id)} + if path == "users": + entity["roles"] = [ + {"id": "default", "name": "DefaultBotRole"}, + {"id": str(ROLE_ID), "name": "GdaMetadataTableProjectionRole"}, + ] + if path == "bots": + entity["botUser"] = { + "id": str(USER_ID), + "name": "gda-metadata-table-projection", + } + self.entities[path][body["name"]] = entity + return self._response(201, entity) + + if method == "DELETE" and is_admin: + collection, entity_id = path.split("/", 1) + for name, entity in list(self.entities.get(collection, {}).items()): + if entity.get("id") == entity_id: + del self.entities[collection][name] + return self._response(200, {}) + return self._response(404, {"code": 404}) + return self._response(404, {"code": 404}) + + +def test_http_rehearsal_uses_bounded_identity_rotates_revokes_and_cleans_up(): + profile = identity.load_profile() + provider = _OpenMetadataMock() + rehearsal = identity.OpenMetadataIdentityRehearsal( + base_url="http://openmetadata.test/api/v1", + username="admin@open-metadata.org", + password=SecretStr("local-bootstrap"), + transport=httpx.MockTransport(provider), + ) + + try: + result = rehearsal.execute(profile) + finally: + rehearsal.close() + + assert result["allowed_probe"]["create_status"] == 201 + assert result["denied_probe"]["status"] == 403 + assert result["jwt_lifecycle"]["old_after_rotation_status"] == 401 + assert result["jwt_lifecycle"]["rotated_authentication_status"] == 200 + assert result["jwt_lifecycle"]["after_revocation_status"] == 401 + assert result["cleanup"]["all_rehearsal_objects_absent"] is True + rendered = json.dumps(result) + assert "local-jwt-one" not in rendered + assert "local-jwt-two" not in rendered + assert "bootstrap-access" not in rendered diff --git a/docs/architecture-decisions/adr-051-local-openmetadata-bounded-provider-identity.md b/docs/architecture-decisions/adr-051-local-openmetadata-bounded-provider-identity.md new file mode 100644 index 00000000..798f974d --- /dev/null +++ b/docs/architecture-decisions/adr-051-local-openmetadata-bounded-provider-identity.md @@ -0,0 +1,94 @@ +# ADR-051: Local OpenMetadata Bounded Provider Identity + +**Status**: Accepted + +**Date**: 2026-07-28 + +**Decision owners**: Metadata Platform, Data Governance, Security, Platform Architecture + +**Related decisions**: [ADR-006](adr-006-openmetadata-governance-and-active-metadata-platform.md) · [ADR-024](adr-024-dispatch-authorization-evidence.md) · [ADR-037](adr-037-local-metadata-fabric-foundation-sandbox.md) · [ADR-048](adr-048-local-authorized-metadata-fabric-ingestion-replay.md) · [ADR-050](adr-050-idempotent-openlineage-http-delivery.md) + +## Context + +M3-2 mutated the local providers with the OpenMetadata bootstrap administrator and unauthenticated Gravitino. M3-3 and M3-4 then proved binding persistence and idempotent lineage delivery without calling either provider. The next identity slice must answer a narrower question before protected OIDC is available: can OpenMetadata `1.13.1` provision a dedicated non-admin bot, restrict the project-specific grant to the required resource operation, reject an administrative mutation, and invalidate old credentials after rotation and revocation? + +The local Gravitino `1.3.0` deployment uses the `simple` authenticator with `gravitino.authorization.enable=false`. It cannot support an honest equivalent claim in this sandbox. Kubernetes ServiceAccounts also have token automount disabled and are not used to authenticate the local HTTP caller. This decision therefore separates a bounded OpenMetadata provider-native result from protected workload identity and the unverified Gravitino boundary. + +## Options Considered + +| Option | Benefit | Cost/risk | Decision | +|---|---|---|---| +| Reuse `IngestionBotRole` | Provider-native and immediately available | Grants Create/Delete/EditAll across all resources and cannot prove a bounded project role | Rejected | +| Keep the bootstrap administrator in the ingestion path | Matches M3-2 | No least-privilege or credential-lifecycle evidence | Rejected | +| Create a dedicated OpenMetadata policy, role and JWT bot for one ephemeral probe | Exercises real provider authorization, rotation and revocation | OpenMetadata still attaches its mandatory `DefaultBotRole`; bootstrap admin provisions the identity | Adopted for local OpenMetadata only | +| Enable Gravitino access control without a validated identity backend | Appears to complete both providers | Would turn configuration presence into a false authentication claim | Rejected | + +## Decision + +### 1. The bootstrap administrator is only an ephemeral provisioner + +The local Basic administrator may create and remove the dedicated policy, role, bot user and bot. Its access value remains in memory and never enters evidence or exceptions. The bot principal must report `isAdmin=false`, `isBot=true` and exactly two roles: provider-mandatory `DefaultBotRole` plus `GdaMetadataTableProjectionRole`. Any pre-existing rehearsal object blocks before mutation. + +The administrator is not described as minimum privilege. Kubernetes ServiceAccount identity is observed only to confirm the provider workload still uses `openmetadata` with token automount disabled; it is not presented as the HTTP caller. + +### 2. The only project-specific grant is `Create` on `table` + +`GdaMetadataTableProjectionPolicy` contains exactly one allow rule: operation `Create`, resource `table`. Broad resources, `EditAll`, `Delete`, `All`, extra roles or a changed policy block evidence. OpenMetadata mandatorily attaches `DefaultBotRole`; therefore `local_openmetadata_minimum_privilege_verified=true` is explicitly scoped to `dedicated_table_create_grant_with_provider_mandatory_default_role`. It does not mean that all provider defaults contain only this operation. + +The positive probe creates and reads back `gda_lakehouse.land_use.published.gda_provider_identity_probe`. The same bot then attempts to create `GdaUnauthorizedPolicyProbe`; only HTTP 403 passes. OpenMetadata HTTP 200 and 201 are both valid successful create outcomes, while an unexpectedly accepted denied probe is tracked for administrator cleanup and blocks evidence. + +### 3. Rotation and revocation must invalidate the exercised JWT + +The bot first authenticates with a one-hour JWT. The administrator rotates it through the provider API. The old value must then return 401, while the new value must authenticate as the same provider user. After explicit revocation, the new value must also return 401. No JWT, password, Secret, credential hash or provider error body is recorded. + +This is a local provider-native static JWT lifecycle. It is not OIDC federation, short-lived workload exchange, external secret delivery, protected provisioning, automated rotation scheduling or production IAM. + +### 4. The rehearsal leaves no provider object behind + +The probe table, bot, user, role, dedicated policy and any unexpectedly created denial probe are removed in dependency order. Final natural-key lookups must all return absent, and the loopback port-forward must stop before evidence can pass. The historical M3-2 projection and GDA control data are not mutated. + +### 5. Gravitino remains blocked + +Evidence fixes `gravitino_authentication_verified=false`, `provider_minimum_privilege_verified=false`, `protected_workload_identity_verified=false`, `oidc_verified=false`, `production_identity_verified=false` and `production_ready=false`. Gravitino requires a separately selected authenticated backend, enabled access control, durable catalog profile and positive/negative provider tests before any of those boundaries may change. + +## Verification + +The local Docker Desktop rehearsal records: + +- OpenMetadata workload `openmetadata` at `1.13.1`, ServiceAccount `openmetadata`, token automount disabled and one ready replica; +- non-admin bot user UUID `4b492bb1-b32d-4f05-9cb6-878a6a2bce45` and bot UUID `94ed8f54-d1eb-44a4-a28b-34f19a5c5505`; +- dedicated policy UUID `df3b5584-6185-49b9-8e29-e43fbadd174c` and role UUID `e41ffd2b-2bea-41a0-beaa-22fe0df9f21c`; +- table create HTTP 201, read-back HTTP 200 and provider table UUID `740124bc-b972-419e-9990-5b071ba8ca1e`; +- policy create HTTP 403; +- old JWT after rotation HTTP 401, rotated JWT HTTP 200 and rotated JWT after revocation HTTP 401; +- all six natural-key cleanup checks passed and the port-forward stopped; +- evidence fingerprint `61b6a3429ae948f563bfc2bd012d8b586be581704cec646fd5e74b991243f03f`. + +Focused tests cover strict profile parsing, broad policy and production-claim rejection, sensitive-field rejection, role/policy/denial/lifecycle/cleanup drift, evidence tampering and a stateful HTTP transport that exercises create, rotate, revoke and cleanup without projecting JWT values. + +## Claim Boundary + +Allowed now: + +- one ephemeral local OpenMetadata bot used a dedicated `table/Create` grant and was denied `policy/Create`; +- provider JWT rotation invalidated the old value and explicit revocation invalidated the replacement; +- the bounded identity and probe objects were removed after the rehearsal; +- `local_openmetadata_bounded_identity_verified`, scoped local minimum privilege, JWT rotation and JWT revocation are true for this observation. + +Fixed false now: + +- minimum privilege across both metadata providers; +- protected workload identity, Kubernetes-to-provider identity exchange and OIDC; +- Gravitino authentication, authorization and durable catalog conformance; +- production credential provisioning, storage, delivery, scheduled rotation and incident recovery; +- production ingestion and `production_ready`. + +## Consequences + +**Positive**: M3 no longer relies only on bootstrap-admin behavior to understand OpenMetadata authorization. A dedicated provider identity now has a real allow result, administrative denial, rotation result, revocation result and cleanup proof. + +**Negative**: the test identity is provisioned by a local administrator and OpenMetadata adds a mandatory default bot role. The identity is ephemeral and is not wired into the retained M3-2 projection or a protected worker. + +**Mitigation**: keep the overall provider and production claims false. In a protected environment, replace local Basic provisioning with an attested identity workflow, bind the exact workload and deployment revision, deliver short-lived credentials without repository or evidence exposure, repeat positive/negative tests, and independently enable and validate Gravitino authentication and access control. + +**Revisit trigger**: change this contract when OpenMetadata provider defaults, JWT APIs, protected OIDC exchange or Gravitino's selected authentication and authorization model changes. diff --git a/docs/evidence/metadata-fabric-provider-identity-2026-07-28.json b/docs/evidence/metadata-fabric-provider-identity-2026-07-28.json new file mode 100644 index 00000000..a4439e65 --- /dev/null +++ b/docs/evidence/metadata-fabric-provider-identity-2026-07-28.json @@ -0,0 +1,147 @@ +{ + "errors": [], + "evidence_fingerprint": "61b6a3429ae948f563bfc2bd012d8b586be581704cec646fd5e74b991243f03f", + "gravitino_authentication_verified": false, + "local_openmetadata_bounded_identity_verified": true, + "local_openmetadata_jwt_revocation_verified": true, + "local_openmetadata_jwt_rotation_verified": true, + "local_openmetadata_minimum_privilege_verified": true, + "observation": { + "contract": { + "contract_fingerprint": "707f90aa0b7472380a43bf5aac0a6542db619d06da95c7d0536434f74852965e", + "local_static_contract_verified": true + }, + "gravitino": { + "access_control_enabled": false, + "authentication_verified": false, + "authenticator": "simple", + "version": "1.3.0" + }, + "observed_at": "2026-07-28T13:29:46.197981+00:00", + "openmetadata": { + "allowed_probe": { + "create_status": 201, + "entity_id": "740124bc-b972-419e-9990-5b071ba8ca1e", + "fully_qualified_name": "gda_lakehouse.land_use.published.gda_provider_identity_probe", + "operation": "Create", + "read_status": 200, + "resource": "table" + }, + "bot": { + "id": "94ed8f54-d1eb-44a4-a28b-34f19a5c5505", + "name": "gda-metadata-table-projection-bot", + "provider": "automation" + }, + "cleanup": { + "absence": { + "bot": true, + "policy": true, + "role": true, + "table": true, + "unauthorized_policy": true, + "user": true + }, + "all_rehearsal_objects_absent": true, + "delete_statuses": { + "bot": 200, + "policy": 200, + "role": 200, + "table": 200, + "user": 404 + }, + "jwt_revocation_attempted": true + }, + "denied_probe": { + "operation": "Create", + "resource": "policy", + "status": 403 + }, + "jwt_lifecycle": { + "after_revocation_status": 401, + "expiry": "OneHour", + "initial_authentication_status": 200, + "old_after_rotation_status": 401, + "rotated_authentication_status": 200, + "rotated_principal_matches": true, + "sensitive_material_recorded": false + }, + "policy": { + "enabled": true, + "id": "df3b5584-6185-49b9-8e29-e43fbadd174c", + "name": "GdaMetadataTableProjectionPolicy", + "rules": [ + { + "effect": "allow", + "name": "GdaMetadataTableCreate", + "operations": [ + "Create" + ], + "resources": [ + "table" + ] + } + ] + }, + "principal": { + "effective_roles": [ + "DefaultBotRole", + "GdaMetadataTableProjectionRole" + ], + "id": "4b492bb1-b32d-4f05-9cb6-878a6a2bce45", + "is_admin": false, + "is_bot": true, + "minimum_privilege_scope": "dedicated_table_create_grant_with_provider_mandatory_default_role", + "name": "gda-metadata-table-projection", + "provider_mandatory_default_role_inherited": true + }, + "provisioner": { + "auth_mode": "local_basic_bootstrap_provisioner", + "bootstrap_admin_used": true, + "minimum_privilege": false + }, + "role": { + "id": "e41ffd2b-2bea-41a0-beaa-22fe0df9f21c", + "mandatory_default_role": "DefaultBotRole", + "name": "GdaMetadataTableProjectionRole", + "policy_ids": [ + "df3b5584-6185-49b9-8e29-e43fbadd174c" + ] + } + }, + "runtime": { + "context": "docker-desktop", + "namespace": { + "name": "gda-metadata-sandbox", + "uid": "812c9c5c-9ce2-409e-a6a8-a3f24785aa0c" + }, + "service": { + "name": "openmetadata", + "type": "ClusterIP", + "uid": "c903a96c-9aa9-420c-93d1-d2224df0cc85" + }, + "workload": { + "image": "docker.getcollate.io/openmetadata/server:1.13.1", + "kind": "Deployment", + "name": "openmetadata", + "ready_replicas": 1, + "service_account": "openmetadata", + "service_account_automount_disabled": true, + "uid": "39bba7aa-b48e-4f5c-a9a8-c9cc9ca4ec1f" + } + }, + "runtime_checks": { + "all_port_forwards_stopped": true, + "kubernetes_service_account_used_for_provider_login": false, + "provider_objects_retained": false, + "sensitive_material_recorded": false + }, + "schema": "gda.metadata_fabric_provider_identity_observation.v1" + }, + "oidc_verified": false, + "production_identity_verified": false, + "production_ready": false, + "protected_workload_identity_verified": false, + "provider_minimum_privilege_verified": false, + "schema": "gda.metadata_fabric_provider_identity_evidence.v1", + "status": "local_openmetadata_bounded_identity_verified" +} diff --git a/docs/roadmap-ar0-platform-truth-2026-07-24.md b/docs/roadmap-ar0-platform-truth-2026-07-24.md index 507daf64..c89e1dd9 100644 --- a/docs/roadmap-ar0-platform-truth-2026-07-24.md +++ b/docs/roadmap-ar0-platform-truth-2026-07-24.md @@ -199,7 +199,7 @@ Temporal 继续保持目标组件状态，不在这一包并行接入。OpenMeta 当前完成仅指本地合同、授权 evidence、outbox/callback 代码、数据库成功终局门、托管 worker 代码、默认关闭的部署模板及离线 activation/release preflight、candidate/registry/provenance/artifact-release/live observation evidence gate、合成 golden slice、定向测试、真实 PostgreSQL 16 事务边界和 canonical mainline 治理。`candidate_validated`、`registry_subject_bound`、本地合成 `provenance_verified`、`ready_for_activation`、`ready_for_staging_apply`、`verified_for_staging_apply` 和本地 live collection 都不等于真实镜像已 attested 或 staging 已部署；真实 IAM/OIDC 与 service token 生命周期、首次 GHCR publish/verify、真实 provenance artifact verify、registry-backed live staging revision、worker/callback 扩容运行、golden slice staging 运行链、受保护 release/live evidence provenance、独立 DolphinScheduler metadata PostgreSQL 和真实数据终局证据仍属于 4.7 后续切片。 -### 4.8 Metadata Fabric Bridge M1 + M2 + M3-4（本地 OpenLineage wire delivery 已验证） +### 4.8 Metadata Fabric Bridge M1 + M2 + M3-5（本地 OpenMetadata 受限身份已验证） 第八块回到 AR-1 的 metadata control plane，以 [ADR-036](architecture-decisions/adr-036-read-only-metadata-fabric-bridge-contract.md) 固定 OpenMetadata + Gravitino + GDA Control Ledger 的首条 table slice： @@ -224,8 +224,9 @@ Temporal 继续保持目标组件状态，不在这一包并行接入。OpenMeta 19. [ADR-048](architecture-decisions/adr-048-local-authorized-metadata-fabric-ingestion-replay.md) 已将同一 plan 绑定到精确 PolicyDecision、独立 ApprovalRecord 与 execution-plan Artifact，在本地 OpenMetadata/Gravitino 以 natural key 创建并 read-back；首次 apply 创建 8 个目标层级对象，第二次 replay 为 `no_op/0 mutations`，OpenMetadata provider UUID 与 binding candidate 均来自真实回读。apply plan fingerprint 为 `241cb2018c093f76378d265ab8fb617d161c1be7bd4effa6fad361e9db7522c4`，authorization fingerprint 为 `7bc8f577cbdea8d9979b2606278a52176cc2d723a6159c4e1f35ada0f5bb6db0`，evidence fingerprint 为 `3d5fb07267680520d2f03bf27f354787b7253210eb93ab85aae83d5f5a714dbe`。partial inventory 在 mutation 前 blocked，第二 provider 失败会反向补偿当前 attempt 创建的对象；binding candidate 未写 GDA Control。 20. [ADR-049](architecture-decisions/adr-049-tenant-scoped-metadata-fabric-binding-ledger.md) 已新增 migration 097 与 `PlatformGateway` binding commit/read：真实 provider refs、target/source/definition version、execution-plan、精确 PolicyDecision、独立 Approval 和 provider evidence 必须在同一 tenant 下完整匹配后才可追加。空临时 PostgreSQL 首次提交为 `created=true`、第二次精确 replay 为 `created=false`，FORCE RLS、跨租户不可见和 gateway 无 UPDATE/DELETE 均通过；binding UUID 为 `9580cd65-9fd9-5216-90a5-1fd6837e6cfb`，record SHA 为 `19bdbddedc27d2ed8a35119e8f065a47a02345f9bbd3a51075856cb9587f4176`，evidence SHA 为 `518bfed363aba34e539ada19ea1dc708bacc9eba6578ccab165d11bccfc05223`。M3-3 不调用 provider、不写 legacy，也不覆盖含 synthetic UUID 的既有 Resource。 21. [ADR-050](architecture-decisions/adr-050-idempotent-openlineage-http-delivery.md) 已新增 migration 098 与 tenant-scoped lineage outbox；Gateway 只有在 M3-3 binding、execution-plan 和完整 M3-1 source plan 精确匹配后才可 enqueue。真实 loopback HTTP 演练让接收端先提交事件再返回 503，第二次以同一 `Idempotency-Key` 重发并返回 duplicate 200；共 2 个 wire requests、1 次唯一接受、最终 2 attempts/delivered，完成项不再 claim。delivery UUID 为 `49a54408-b3a8-5843-a27d-6395c080af99`，event SHA 为 `4929e51c4126e09415a9fc1578c9401077c5d7c374294e70deeebd29c8216dd2`，evidence SHA 为 `8fa87a34a39b900df0673f11d0301c9f5155ce64ff9502125478ec59a3f0fdb6`。该结论是本地 `at_least_once_with_receiver_idempotency`，不是网络 exactly-once 或生产 receiver 证明。 +22. [ADR-051](architecture-decisions/adr-051-local-openmetadata-bounded-provider-identity.md) 已在 OpenMetadata `1.13.1` 创建临时非管理员 bot，effective roles 精确为 provider 强制 `DefaultBotRole` 加项目专用 role；项目新增 policy 仅允许 `table/Create`。bot 创建/read-back 临时 table 分别返回 201/200，创建 policy 返回 403；JWT 轮换后旧值返回 401，新值返回 200，吊销后新值返回 401。table、bot、user、role、policy 与 denial probe 最终均不存在，evidence SHA 为 `61b6a3429ae948f563bfc2bd012d8b586be581704cec646fd5e74b991243f03f`。该 scoped local 结论不包含 Gravitino、OIDC、Kubernetes-to-provider identity exchange 或生产 credential delivery。 -此处 M1 只证明静态合同和只读 HTTP 边界；M2a 只证明本地 live foundation 与 PVC 重挂载连续性；M2b-1/M2b-2 分别限定在同集群新 PVC 和同集群隔离 repository；M2b-3 的 `local_cross_cluster_recovery_verified=true` 只限定在 `local_same_host_distinct_kubernetes_clusters_external_s3_repository`；M2c-1/M2c-2/M2c-3 分别限定本地 provider metrics、临时双周期 OTel 和单 job scrape recovery；M2c-4/M2d-2 只证明 production observability/NetworkPolicy profile 与 attestation 合同可校验；M2d-1 只证明本地两节点 kindnet 的隔离合成流量；M3-1 的 terminal evidence 与 M3-2 的 PolicyDecision/Approval 仍是 deterministic local fixtures。M3-2 只把 projection 写入本地 provider 并证明 retained target 的单次零写入 replay；M3-3 只把该本地 evidence 对应的 binding 写入临时 GDA Control 账本；M3-4 只向无认证 loopback receiver 发送精确 candidate 并验证 503 后幂等恢复，不再次调用 metadata provider。OpenMetadata 使用 bootstrap admin，Gravitino 未认证且 catalog backend 为 memory，restart persistence 未验证。生产持久 binding、ResourceVersion 和 legacy authority 都未写入；生产最小权限、OIDC、持久 catalog、TLS、tenant isolation、真实 receiver/alert/SLO、受保护 provider policy、生产 OpenLineage、生产 ingest/conformance、两项 production gate 和 `production_ready` 仍为 `false`。 +此处 M1 只证明静态合同和只读 HTTP 边界；M2a 只证明本地 live foundation 与 PVC 重挂载连续性；M2b-1/M2b-2 分别限定在同集群新 PVC 和同集群隔离 repository；M2b-3 的 `local_cross_cluster_recovery_verified=true` 只限定在 `local_same_host_distinct_kubernetes_clusters_external_s3_repository`；M2c-1/M2c-2/M2c-3 分别限定本地 provider metrics、临时双周期 OTel 和单 job scrape recovery；M2c-4/M2d-2 只证明 production observability/NetworkPolicy profile 与 attestation 合同可校验；M2d-1 只证明本地两节点 kindnet 的隔离合成流量；M3-1 的 terminal evidence 与 M3-2 的 PolicyDecision/Approval 仍是 deterministic local fixtures。M3-2 只把 projection 写入本地 provider 并证明 retained target 的单次零写入 replay；M3-3 只把该本地 evidence 对应的 binding 写入临时 GDA Control 账本；M3-4 只向无认证 loopback receiver 发送精确 candidate 并验证 503 后幂等恢复；M3-5 只证明 OpenMetadata 在 provider 强制默认 role 之上新增的 grant 限定为 `table/Create`，以及本地 JWT 轮换/吊销和越权拒绝。M3-2 ingestion 仍使用 bootstrap admin，Gravitino 未认证且 catalog backend 为 memory，restart persistence 未验证。生产持久 binding、ResourceVersion 和 legacy authority 都未写入；双 provider/生产最小权限、protected workload identity、OIDC、持久 catalog、TLS、tenant isolation、真实 receiver/alert/SLO、受保护 provider policy、生产 OpenLineage、生产 ingest/conformance、两项 production gate 和 `production_ready` 仍为 `false`。 ## 5. 重新评估条件 diff --git a/docs/roadmap.md b/docs/roadmap.md index decfff47..218e5394 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -417,7 +417,7 @@ AR-0 Architecture / Schema / Runtime Truth - Web、DolphinScheduler、OpenMetadata、Gravitino、worker 和 bridge 重启后不丢版本、产品、血缘、PlatformRun correlation 或审批状态。 - DataOps 状态可从 DolphinScheduler/OpenMetadata/Gravitino/GDA evidence 恢复；任何 release/deployment 都能从事件、Artifact、评测、策略、incident 和 rollback pointer 重放。Temporal durable recovery 在 AR-5/AR-7 单独验收。 -当前 Metadata Fabric 证据边界：M1 只读 bridge 合同已验证；ADR-037 至 ADR-046 分别覆盖本地 foundation/recovery/metrics/network-policy 演练与 production readiness contracts；ADR-047 的 M3-1 已建立 deterministic projection plan；ADR-048 的 M3-2 已用 exact local PolicyDecision/Approval 对本地 OpenMetadata/Gravitino 执行 natural-key create/read-back，并证明第二次 replay 为 `no_op/0 mutations`。M3-2 使用 OpenMetadata bootstrap admin、未认证 Gravitino 与 memory catalog，binding candidate 不写 GDA Control，OpenLineage 不发送；因此只允许 `local_live_provider_ingestion_verified=true`，不证明 provider 最小权限、OIDC、持久 catalog、生产 ingestion/conformance、live lineage 或 `production_ready`。 +当前 Metadata Fabric 证据边界：M1 只读 bridge 合同已验证；ADR-037 至 ADR-046 分别覆盖本地 foundation/recovery/metrics/network-policy 演练与 production readiness contracts；ADR-047 至 ADR-050 已依次建立 deterministic projection plan、本地双 provider replay、tenant-scoped binding ledger 与本地 OpenLineage 幂等 wire delivery；ADR-051 又以临时非管理员 OpenMetadata bot 证明项目专用 grant 只有 `table/Create`、`policy/Create` 被 403 拒绝，且 JWT 轮换/吊销后旧值/当前值均返回 401。这个 `local_openmetadata_minimum_privilege_verified=true` 明确限定在 provider 强制 `DefaultBotRole` 之上的项目新增 grant；M3-2 ingestion 仍使用 bootstrap admin，Gravitino 仍未认证且使用 memory catalog。因此 `provider_minimum_privilege_verified`、protected workload identity、OIDC、持久 catalog、生产 ingestion/conformance、生产 lineage receiver 与 `production_ready` 仍为 false。 ### AR-2 — Source, Ingestion and Geospatial Lakehouse Vertical Slice（P0） diff --git a/docs/system-of-record-matrix-2026-07-24.md b/docs/system-of-record-matrix-2026-07-24.md index 2da1f2a1..4dd55196 100644 --- a/docs/system-of-record-matrix-2026-07-24.md +++ b/docs/system-of-record-matrix-2026-07-24.md @@ -2,9 +2,9 @@ 日期：2026-07-28 -阶段：AR-0 `in_progress`；AR-1 gateway、成功终局 evidence gate、DolphinScheduler adapter sandbox POC、Metadata Fabric M1/M2、M2c-4/M2d-2 production readiness contracts、M3-1/M3-2、M3-3 local binding ledger 与 M3-4 local OpenLineage wire delivery 已验证，生产 provider ingestion、生产观测、生产 policy/tenant isolation 和生产切换仍 `in_progress` +阶段：AR-0 `in_progress`；AR-1 gateway、成功终局 evidence gate、DolphinScheduler adapter sandbox POC、Metadata Fabric M1/M2、M2c-4/M2d-2 production readiness contracts、M3-1/M3-2、M3-3 local binding ledger、M3-4 local OpenLineage wire delivery 与 M3-5 local OpenMetadata bounded identity 已验证，生产 provider ingestion、生产观测、生产 policy/tenant isolation 和生产切换仍 `in_progress` -适用分支：`feat/ar1-metadata-fabric-openlineage-delivery` +适用分支：`feat/ar1-metadata-fabric-provider-identity` ## 判定规则 @@ -26,7 +26,7 @@ | 在线空间数据 | PostGIS 业务表是当前编辑/查询事实，部分临时表混入 | Martin MVT、API JSON、导出文件 | 已批准 DataProductVersion 物化到 PostGIS；不能由瓦片或临时表反向定义产品版本 | GIS/Data Platform | AR-2 -> AR-4 | | 数据资产身份与版本 | `gda_control.resource/resource_version` 已实现 identity、hash、predecessor、tenant FK 和幂等 gateway 写入；`agent_data_assets`、`agent_asset_versions` 仍是兼容写路径 | UI catalog、search index、STAC | GDA ledger 管身份与版本绑定；旧行只有在 tenant、authority identity、checksum 和 version evidence 完整时才可形成 eligible plan；OpenMetadata 管治理目录，Gravitino 管技术对象映射 | Metadata Platform | AR-1 gateway 已验证 -> 生产切换待验收 | | 技术元数据 | M1 已冻结 Gravitino table ref/reconciliation；M2 已验证本地 foundation/recovery/metrics/policy 和 production readiness contracts；M3-1 固定 technical projection intent，M3-2 已在 Gravitino memory catalog 创建/read-back，M3-3 将验证后的 ref 追加到 tenant-scoped 本地 binding ledger | harvester 结果、合成 response、本地 sandbox/recovery/metrics/policy/ingestion observation、projection plan、provider evidence、binding ledger 与 readiness report | 源系统技术对象是原始证据；Gravitino 映射并联邦，不能覆盖业务 ResourceVersion；GDA binding ledger 只记录已验证关系，本地 memory catalog/evidence 不得冒充生产持久技术权威 | Metadata Platform | AR-1 M1/M2 + M3-3 local ledger 已验证 -> 认证持久 catalog/生产 conformance 待执行 | -| 治理目录 | M1 已冻结 OpenMetadata table ref/reconciliation；M2 已验证本地 foundation/recovery/metrics/policy 和 production readiness contracts；M3-2 已用 bootstrap admin 创建目标并回读真实 UUID；M3-3 将该 UUID 经 evidence gate 追加到本地 GDA binding ledger；M3-4 将精确 OpenLineage candidate 经 outbox 投递到本地 HTTP receiver | 搜索/页面视图、合成 response、本地 sandbox/recovery/metrics/policy/ingestion observation、projection/provider evidence、binding ledger、lineage outbox/receipt 与 OpenLineage event | OpenMetadata 为 owner/glossary/classification/quality discoverability 权威；GDA ledger 保留审批/provider identity，outbox 只拥有投递状态，receiver 拥有接收状态；均不反写 ResourceVersion | Governance | AR-1 M1/M2 + M3-4 local wire delivery 已验证 -> 最小权限 ingestion/生产持久 binding/受保护 production receiver 待执行 | +| 治理目录 | M1 已冻结 OpenMetadata table ref/reconciliation；M2 已验证本地 foundation/recovery/metrics/policy 和 production readiness contracts；M3-2 已用 bootstrap admin 创建目标并回读真实 UUID；M3-3 将该 UUID 经 evidence gate 追加到本地 GDA binding ledger；M3-4 将精确 OpenLineage candidate 经 outbox 投递到本地 HTTP receiver；M3-5 已验证临时非管理员 bot 的 scoped `table/Create` grant、policy-create 拒绝及 JWT 轮换/吊销 | 搜索/页面视图、合成 response、本地 sandbox/recovery/metrics/policy/ingestion/identity observation、projection/provider evidence、binding ledger、lineage outbox/receipt 与 OpenLineage event | OpenMetadata 为 owner/glossary/classification/quality discoverability 权威；GDA ledger 保留审批/provider identity，outbox 只拥有投递状态，receiver 拥有接收状态；均不反写 ResourceVersion | Governance | AR-1 M1/M2 + M3-5 local bounded identity 已验证 -> protected identity ingestion/生产持久 binding/受保护 production receiver 待执行 | | 血缘 | `gda_control.lineage_event` 已实现 immutable version edge 和幂等 gateway ingest；`agent_asset_lineage` 旧记录仍是可变 asset edge | OpenMetadata lineage graph、UI DAG | 只有 source/target ResourceVersion 与 event checksum 证据完整的旧记录可形成 eligible plan；目录图只作可重建投影 | Data Platform | AR-1 gateway 已验证 -> adapter 待接入 | | Definition | `gda_control.platform_definition_version` 已绑定 definition ResourceVersion、完整逻辑 hash 和原子 gateway registration；3.4.2 adapter 可编译、创建并上线 provider DAG；binding 已以 append-only `execution_plan` Artifact 持久化并可按 tenant + artifact UUID 读取，旧 workflow/template/YAML 仍在写入 | 编辑器状态、DolphinScheduler DAG/definition | 旧 workflow 必须规范化并完整 hash 后才可形成 PlatformDefinitionVersion；provider binding 作为 ExecutionPlanArtifact/evidence，不可反写 definition | DataOps | AR-1 binding persistence 代码已验证 -> staging 调用链待验收 | | Run 最终状态 | `gda_control.platform_run/event` 已实现受控 submit/read/CAS；通用 transition 已禁止 `succeeded`，专用数据库 finalizer 只接受精确 workload、DolphinScheduler success observation、内容匹配 output、独立 passed QualityResult/evidence 和 input-to-output lineage；adapter standalone API path 已验证，但端到端 staging 尚未完成，legacy 路径继续运行 | Redis progress、日志、DolphinScheduler state、attempt observation | 旧 run 到 PlatformRun 永久 prohibited；已有 PlatformRun correlation 时才可转为 observation；provider 终态只进入 `reconciling`，ledger 经证据门唯一裁决成功 | DataOps/AgentOps | AR-1 success authority 本地/PostgreSQL 已验证 -> staging/生产切换待验收 | @@ -34,7 +34,7 @@ | 事件交付 | Standards outbox 已数据库耐久；`gda_control.platform_command_outbox` 已为 DolphinScheduler dispatch/reconcile 提供 tenant RLS、lease claim、幂等 callback、薄 consumer library 和 managed worker process；其他 WebSocket/bot/feedback 多为 best effort | command delivery status、消费者 claim、worker status JSON、WebSocket 消息 | command/event 与源事实同事务入 outbox，幂等 consumer 交付；worker status、outbox 状态、缓存或 socket 都不是 Run/业务权威 | Platform/Integrations | AR-1 command delivery/worker 代码已验证 -> staging worker/callback 待部署 | | 质量结果 | `gda_control.quality_result` 已提供 tenant RLS、append-only gateway 写入，绑定 Run、output ResourceVersion、rule version、verdict、metrics、evidence Artifact 和独立 evaluator；standards、QC、MMFE 专项结果仍未迁移 | dashboard、OpenMetadata quality summary | GDA ledger 保存产品终局所需的不可变 verdict/evidence；OpenMetadata 与 UI 只作可重建发现投影；旧结果缺稳定版本和证据时不得升级为终局依据 | Governance/DataOps | AR-1 最小成功证据已验证 -> 真实规则/staging 待接入 | | 标准与语义定义 | `std_*`、semantic registry 和 YAML 共同存在，生命周期未统一 | prompt/context、搜索索引 | 版本化 Standard/SemanticDefinition 经审批后为权威；Agent context 只消费批准版本 | Governance | AR-1 -> AR-3 | -| 身份与权限 | Chainlit user 可显式绑定 tenant；versioned API 从认证 principal 派生 SubjectContext；`gda_control_gateway` 是 non-login/non-bypass 最小权限角色；Run 可引用强类型 PolicyDecision/Approval Artifact，DolphinScheduler dispatch 已绑定配置的 workload/evaluator 并在 provider 调用前校验证据 | session/cache、前端菜单权限、provider token profile | IdP/workload identity 提供真实 service identity；PolicyDecision/Approval 继续绑定不可变资源与 execution plan；完成 OIDC/IAM provisioning、轮换、吊销和 provider 最小权限后才能生产切换 | Security | AR-1 dispatch authorization 代码已验证 -> staging IAM 待验收 | +| 身份与权限 | Chainlit user 可显式绑定 tenant；versioned API 从认证 principal 派生 SubjectContext；`gda_control_gateway` 是 non-login/non-bypass 最小权限角色；Run 可引用强类型 PolicyDecision/Approval Artifact；M3-5 本地 OpenMetadata bot 已验证 scoped project grant、越权拒绝和 JWT 轮换/吊销，但由 bootstrap admin 临时 provision，Gravitino 未认证 | session/cache、前端菜单权限、本地 provider identity evidence | IdP/workload identity 提供真实 service identity；PolicyDecision/Approval 继续绑定不可变资源与 execution plan；完成受保护 OIDC/IAM provisioning、双 provider 最小权限、secret delivery 和轮换/吊销后才能生产切换 | Security | AR-1 local OpenMetadata identity 已验证 -> protected 双 provider IAM 待验收 | | GIS 服务定义与 active revision | Martin、REST/MVT/STAC endpoints 和配置直接暴露 | Ingress、tile cache、客户端图层 | GIS Service Control Plane 管 Service/Layer/Style/TMS/DeploymentRevision；provider/Gateway 仅执行 | GIS Platform | AR-4 | | 缓存与进度 | Redis、进程内 dict/task map | UI progress、tile/context cache | 永不作为资产、Run、workflow 或产品权威；丢失后必须从 ledger/provider 重建 | SRE/Platform | 持续约束 | | Agent/Prompt/Model bundle | 多个 registry 与 YAML 存在，尚无统一 deployment revision | trace、eval dashboard | AgentSpecBundle + EvaluationBinding + DeploymentRevision；只消费已发布 DataProductVersion | AgentOps | AR-5 | @@ -56,7 +56,7 @@ 11. `platform_command_outbox` 只拥有投递状态；callback 只触发 reconcile，不能把 provider payload 直接写成 PlatformRun 状态或平台终局。 12. QualityResult evaluator 必须是 workload，且成功终局中的 evaluator 不能等于 Run workload；该代码级职责分离不替代生产 IAM。 13. `candidate_validated`、`registry_subject_bound`、GitHub provenance action 成功、CI artifact、离线 preflight、未独立 attested 的 live observation JSON 或人工批准都不能单独授权 production；缺少同一 source revision 的 OCI subject 独立验证、registry/live revision/identity/health/golden-slice 绑定及受保护 provenance 时，promotion 必须失败。 -14. Metadata Fabric M1 只允许 OpenMetadata/Gravitino GET；M2 只执行本地 foundation/recovery/metrics/policy 演练或验证 production readiness profile；M3-1 只从 synthetic terminal evidence 生成 plan/candidate；M3-2 只允许 exact local PolicyDecision/Approval 后向本地 provider 写 projection；M3-3 只将同一 source evidence 经 PlatformGateway 写入临时 append-only binding ledger；M3-4 只经 tenant-scoped outbox 向无认证 loopback receiver 投递精确 candidate，并验证 at-least-once + receiver idempotency。本地 bootstrap admin、未认证 Gravitino、memory catalog、临时 ledger/outbox、loopback receiver、pending profile、合成 attestation 和 local evidence 都不等于生产最小权限/OIDC、持久 catalog/binding、受保护 OpenLineage receiver、tenant isolation、alert/SLO、生产 ingestion/conformance 或生产写权威。 +14. Metadata Fabric M1 只允许 OpenMetadata/Gravitino GET；M2 只执行本地 foundation/recovery/metrics/policy 演练或验证 production readiness profile；M3-1 只从 synthetic terminal evidence 生成 plan/candidate；M3-2 只允许 exact local PolicyDecision/Approval 后向本地 provider 写 projection；M3-3 只将同一 source evidence 经 PlatformGateway 写入临时 append-only binding ledger；M3-4 只经 tenant-scoped outbox 向无认证 loopback receiver 投递精确 candidate，并验证 at-least-once + receiver idempotency；M3-5 只证明临时 OpenMetadata bot 在 provider 强制 `DefaultBotRole` 之上的项目新增 grant 是 `table/Create`，并验证 policy-create 拒绝与本地 JWT 轮换/吊销。本地 bootstrap provisioner、未认证 Gravitino、memory catalog、临时 identity/ledger/outbox、loopback receiver、pending profile、合成 attestation 和 local evidence 都不等于双 provider/生产最小权限、protected workload identity/OIDC、持久 catalog/binding、受保护 OpenLineage receiver、tenant isolation、alert/SLO、生产 ingestion/conformance 或生产写权威。 ## 已建立的 AR-0/AR-1 entry 证据 @@ -84,6 +84,7 @@ - Metadata Fabric M3-2 已将同一 plan 绑定 execution-plan Artifact、精确 PolicyDecision 与独立 ApprovalRecord，在本地 provider 以 natural key 创建后 read-back；首次 apply 为 `created/8 mutations`，第二次为 `no_op/0 mutations`，真实 OpenMetadata UUID 进入未持久化 binding candidate。partial inventory 在 mutation 前阻断，失败 attempt 反向补偿，端口转发已退出且 evidence 不含凭据；evidence fingerprint 为 `3d5fb07267680520d2f03bf27f354787b7253210eb93ab85aae83d5f5a714dbe`。`provider_minimum_privilege_verified=false`、`oidc_verified=false`、`gravitino_authentication_verified=false`、`binding_persisted_to_gda_control=false`、`live_openlineage_emission_verified=false`、`production_ingestion_verified=false`、`production_ready=false`。 - Metadata Fabric M3-3 已新增 tenant-scoped append-only binding ledger，将 M3-2 的真实 OpenMetadata UUID、Gravitino ref、execution-plan、PolicyDecision、Approval 与 provider evidence 经 PlatformGateway 精确校验后落账；空临时 PostgreSQL 首次提交 `created=true`、replay `created=false`，跨租户读取、UPDATE/DELETE 均拒绝。binding SHA 为 `125d7197f05ff9c37999a94d090d123dcf905480b776da0738d9625ab5045598`，record SHA 为 `19bdbddedc27d2ed8a35119e8f065a47a02345f9bbd3a51075856cb9587f4176`，evidence SHA 为 `518bfed363aba34e539ada19ea1dc708bacc9eba6578ccab165d11bccfc05223`。该演练不调用 provider，`provider_minimum_privilege_verified=false`、`oidc_verified=false`、`durable_catalog_verified=false`、`live_openlineage_emission_verified=false`、`production_ingestion_verified=false`、`production_ready=false`。 - Metadata Fabric M3-4 已新增 tenant-scoped lineage outbox、Gateway claim/complete/fail 和严格 loopback HTTP emitter；接收端第一次提交 event 后返回 503，第二次以同一 idempotency key/body 重发并只返回 duplicate 200，形成 2 wire requests、1 unique accept、2 attempts 和不可再 claim 的 delivered receipt。delivery UUID 为 `49a54408-b3a8-5843-a27d-6395c080af99`，event SHA 为 `4929e51c4126e09415a9fc1578c9401077c5d7c374294e70deeebd29c8216dd2`，evidence SHA 为 `8fa87a34a39b900df0673f11d0301c9f5155ce64ff9502125478ec59a3f0fdb6`。`local_wire_openlineage_delivery_verified=true` 仅表示真实本机 HTTP wire；`provider_minimum_privilege_verified=false`、`oidc_verified=false`、`durable_catalog_verified=false`、`production_receiver_verified=false`、`production_ingestion_verified=false`、`production_ready=false`。 +- Metadata Fabric M3-5 已创建临时 OpenMetadata 非管理员 bot，effective roles 只有 provider 强制 `DefaultBotRole` 与项目 role，项目 policy 只有 `table/Create`；table create/read 为 201/200，policy create 为 403，旧 JWT 轮换后为 401，新 JWT 吊销后为 401，六项 provider cleanup 与 port-forward cleanup 全部通过。evidence SHA 为 `61b6a3429ae948f563bfc2bd012d8b586be581704cec646fd5e74b991243f03f`。`local_openmetadata_minimum_privilege_verified=true` 只描述强制 default role 之上的项目新增 grant；`provider_minimum_privilege_verified=false`、`protected_workload_identity_verified=false`、`oidc_verified=false`、`gravitino_authentication_verified=false`、`production_identity_verified=false`、`production_ready=false`。 ## 下一验收证据 @@ -92,5 +93,5 @@ - staging 的 migration role、应用 login membership、连接池 role/tenant 复位、双租户 API 和 success finalization 运行产物； - DolphinScheduler adapter 的真实 IAM/OIDC、service token provisioning/轮换、provider 最小权限、binding artifact staging 接入、managed outbox worker/provider callback 实际扩容部署、唯一 worker ID、status/lease 故障恢复和无双写证据； - 首条真实图斑链对 golden slice 的 output hash、独立质量结果/evidence、血缘、发布 revision 和 rollback 演练； -- OpenMetadata/Gravitino 的 source host/cluster 外生产 backup account/bucket、KMS/TLS/workload identity、PITR/source-loss recovery、RPO/RTO、OIDC、受保护环境 provider NetworkPolicy/tenant isolation、upgrade/rollback、registry provenance、持续 metrics backend/retention/query、真实 alert delivery/SLO owner/runbook，以及受保护 PolicyDecision/Approval、最小权限 provider ingestion、生产持久 binding、受保护 production OpenLineage receiver、无双写 read-back 和 conformance；M1 fixture、M2 本地 evidence/readiness contracts、M3-1 projection candidate、M3-2 local replay、M3-3 临时 binding ledger 与 M3-4 loopback delivery 均不计入生产退出门； +- OpenMetadata/Gravitino 的 source host/cluster 外生产 backup account/bucket、KMS/TLS/workload identity、PITR/source-loss recovery、RPO/RTO、OIDC、受保护环境 provider NetworkPolicy/tenant isolation、upgrade/rollback、registry provenance、持续 metrics backend/retention/query、真实 alert delivery/SLO owner/runbook，以及受保护 PolicyDecision/Approval、双 provider 最小权限 ingestion、生产持久 binding、受保护 production OpenLineage receiver、无双写 read-back 和 conformance；M1 fixture、M2 本地 evidence/readiness contracts、M3-1 projection candidate、M3-2 local replay、M3-3 临时 binding ledger、M3-4 loopback delivery 与 M3-5 本地临时 OpenMetadata identity 均不计入生产退出门； - DolphinScheduler/Temporal sandbox 的独立数据库、备份恢复、身份、版本和升级责任证明；DolphinScheduler standalone/H2 不计入此退出门。 diff --git a/scripts/metadata-fabric-provider-identity.sh b/scripts/metadata-fabric-provider-identity.sh new file mode 100755 index 00000000..cf70775f --- /dev/null +++ b/scripts/metadata-fabric-provider-identity.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +COMMON_GIT_DIR="$(git -C "$ROOT" rev-parse --path-format=absolute --git-common-dir 2>/dev/null || true)" +SHARED_ROOT="" +if [ -n "$COMMON_GIT_DIR" ]; then + SHARED_ROOT="$(cd "$COMMON_GIT_DIR/.." && pwd)" +fi + +if [ -n "${PYTHON:-}" ]; then + : +elif [ -x "$ROOT/.venv/bin/python" ]; then + PYTHON="$ROOT/.venv/bin/python" +elif [ -n "$SHARED_ROOT" ] && [ -x "$SHARED_ROOT/.venv/bin/python" ]; then + PYTHON="$SHARED_ROOT/.venv/bin/python" +else + PYTHON="python" +fi + +cd "$ROOT" +exec "$PYTHON" -m data_agent.metadata_fabric_provider_identity "$@"