[CXH-2722] - Add Unity Catalog schema sync - Databricks Connector

README.md

@@ -222,6 +222,7 @@ account
 ├── service principal
 └── metastore
     └── catalog
+        └── schema
 ```
 
 A metastore exposes its privileges as entitlements (`CREATE_CATALOG`,

baton_capabilities.json

@@ -255,6 +255,85 @@
         ]
       }
     },
+    {
+      "resourceType": {
+        "id": "schema",
+        "displayName": "Schema",
+        "traits": [
+          "TRAIT_APP"
+        ],
+        "annotations": [
+          {
+            "@type": "type.googleapis.com/c1.connector.v2.OptInRequired"
+          },
+          {
+            "@type": "type.googleapis.com/c1.connector.v2.CapabilityPermissions",
+            "permissions": [
+              {
+                "permission": "unity-catalog"
+              },
+              {
+                "permission": "scim"
+              },
+              {
+                "permission": "access-management"
+              },
+              {
+                "permission": "provisioning"
+              },
+              {
+                "permission": "Account admin"
+              },
+              {
+                "permission": "Workspace admin"
+              },
+              {
+                "permission": "USE_CATALOG"
+              },
+              {
+                "permission": "USE_SCHEMA"
+              }
+            ]
+          },
+          {
+            "@type": "type.googleapis.com/c1.connector.v2.SkipEntitlements"
+          }
+        ]
+      },
+      "capabilities": [
+        "CAPABILITY_SYNC",
+        "CAPABILITY_PROVISION"
+      ],
+      "permissions": {
+        "permissions": [
+          {
+            "permission": "unity-catalog"
+          },
+          {
+            "permission": "scim"
+          },
+          {
+            "permission": "access-management"
+          },
+          {
+            "permission": "provisioning"
+          },
+          {
+            "permission": "Account admin"
+          },
+          {
+            "permission": "Workspace admin"
+          },
+          {
+            "permission": "USE_CATALOG"
+          },
+          {
+            "permission": "USE_SCHEMA"
+          }
+        ]
+      },
+      "optInRequired": true
+    },
     {
       "resourceType": {
         "id": "service_principal",

docs/connector.mdx

@@ -15,6 +15,7 @@ sidebarTitle: "Databricks"
 | Groups | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> |
 | Metastores | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> |
 | Roles | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> |
+| Schemas | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> |
 | Service principals | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> |
 | Users | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> |
 | Workspaces | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> |
@@ -29,19 +30,19 @@ Revoking the workspace **member** entitlement removes the principal's entire wor
 
 [This connector syncs non-human identities](/product/admin/nhi) and displays them on the **Identities overview** dashboard.
 
-## Metastore and catalog access in C1
+## Metastore, catalog and schema access in C1
 
 The connector syncs Databricks [Unity Catalog](https://docs.databricks.com/aws/en/data-governance/unity-catalog/manage-privileges/privileges)
-metastores and the catalogs in them as C1 resources. Each privilege Databricks
+metastores, the catalogs in them, and the schemas in those catalogs as C1 resources. Each privilege Databricks
 records on one of them (`CREATE_CATALOG` on a metastore, `USE_CATALOG` or
-`SELECT` on a catalog, and so on) becomes an entitlement in C1, so you can
+`SELECT` on a catalog, `USE_SCHEMA` on a schema, and so on) becomes an entitlement in C1, so you can
 review who holds it, request it, and revoke it for users, groups and service
 principals. A grant held by a group is expandable so C1 can resolve the
 group's members as holders. Ownership is shown as a read-only `owner`
 entitlement — it cannot be granted or revoked through C1.
 
 <Note>
-The Metastores and Catalogs resource types are turned off by default. Turn on the ones you want from the connector's configuration page in C1. Catalogs appear under their metastore and use the same OAuth credentials; no additional configuration fields are needed.
+The Metastores, Catalogs and Schemas resource types are turned off by default. Turn on the ones you want from the connector's configuration page in C1. Catalogs appear under their metastore and schemas under their catalog, and all of them use the same OAuth credentials; no additional configuration fields are needed.
 </Note>
 
 <Note>
@@ -51,6 +52,7 @@ The Metastores and Catalogs resource types are turned off by default. Turn on th
 - **MANAGE** and **ALL_PRIVILEGES** are not offered on a metastore, because Databricks rejects both at that level.
 - A catalog is synced when any running workspace in scope attached to its metastore lists it, and the same catalog name is kept once per metastore. A catalog that only excluded workspaces can reach is not synced.
 - The built-in `system` catalog and catalogs Databricks identifies as `SYSTEM_CATALOG` are excluded regardless of ownership. This includes `samples` when Databricks reports that type.
+- `information_schema` is skipped.
 - If a catalog becomes unreachable, access requests can fail until the connector can distinguish deletion from a binding or visibility change. Catalog ownership or **MANAGE** alone cannot prove deletion; that check requires the **Metastore Admin**.
 - Unity Catalog tags are not synced.
 </Note>
@@ -202,6 +204,16 @@ For a discovered catalog that is not excluded by the system-catalog rule above,
 The connector asks every running workspace in scope attached to the metastore, so a catalog bound to one of those workspaces is included. A catalog the service principal cannot see on any of them is not, and ConductorOne removes it and its grants.
 </Warning>
 
+### Let the service principal list every schema
+
+Skip this section if you are not syncing schemas, or if the service principal owns the metastore or the catalog.
+
+Workspace **Admin** does not list every schema, except when the catalog is owned by the workspace admins group (`_workspace_admins_<workspace>_<id>`) and the service principal is a direct Admin of that workspace. On any other catalog the service principal does not own, Databricks returns only the schemas it owns or can use, with HTTP 200. To list every schema, the service principal must own the metastore, own the catalog, be that direct Admin, or hold **MANAGE**, **USE SCHEMA**, or **ALL PRIVILEGES** on the catalog.
+
+<Warning>
+A catalog whose schemas cannot all be listed fails the sync. `system` and `samples` are not a special case of that listing: the catalog sync leaves them out, when their privilege list cannot be proven complete, before their schemas are read.
+</Warning>
+
 **Done.** Here's the set of credentials you'll need when setting up the connector:
 
 - Account ID

docs/docs-info.md

@@ -60,9 +60,9 @@ While developing the connector, please fill out this form. This information is n
 - **Ownership is read-only.** It is shown and cannot be granted or revoked. An owner that is not an account identity (`System user`, `_workspace_admins_…`) is not shown.
 - **No `MANAGE` or `ALL_PRIVILEGES` on a metastore.** Databricks rejects both, so they are not offered.
 - **Read through one running workspace.** Privileges are not served on the account plane. The connector reads each metastore through the first running workspace in scope where it can see every privilege. With no running workspace in scope, the metastore is not synced and ConductorOne removes it. A workspace that is still provisioning fails the sync until it is `RUNNING`. An unreadable assignment list fails the sync and ConductorOne keeps the metastore.
-- **`workspaces` scopes metastores.** An allowlist that names no workspace on the account fails at credential validation, before any sync.
+- **`workspaces` scopes metastores.** An allowlist that names no workspace on the account fails at credential validation, before any sync. A name that is also in `databricks-exclude-workspaces` is reported separately from a name the account does not have.
 - **Catalogs are opt-in.** The resource type is off until it is turned on from the connector's configuration page in C1.
 - **Catalog discovery visits every attached workspace in scope.** The connector asks each running workspace attached to the metastore, including for `ISOLATED` catalogs bound to only some workspaces. Each catalog name is kept once per metastore. A catalog bound only to workspaces the sync excludes is not listed, and ConductorOne removes it with its grants.
 - **System catalogs are excluded regardless of ownership.** The built-in `system` catalog and any catalog with `CatalogType == "SYSTEM_CATALOG"` are excluded, including `samples` when it has that type.
 - **Seeing a catalog needs more than `MANAGE`.** The list holds the catalogs the service principal owns or holds `USE_CATALOG` on, or every catalog when it is the metastore admin. `MANAGE` alone reads a catalog's privileges but does not make it appear.
-- **Not synced yet:** schemas, tables, volumes, functions, registered models, external locations, storage credentials, connections, shares, recipients, and tags.
+- **Not synced yet:** tables, volumes, functions, registered models, external locations, storage credentials, connections, shares, recipients, and tags.

pkg/connector/catalog.go

@@ -41,7 +41,8 @@ const (
 	errorCodeMetastoreGone = "METASTORE_DOES_NOT_EXIST"
 
 	// systemCatalog is the built-in catalog Databricks puts in every metastore.
-	// It is excluded independently of the connector's ownership.
+	// samples is the other one; both are SYSTEM_CATALOG. Both are excluded
+	// independently of the connector's ownership.
 	systemCatalog = "system"
 
 	// workspaceAdminsGroupPrefix starts the name of the system group Databricks
@@ -186,7 +187,10 @@ func catalogResource(
 		nil,
 		rs.WithResourceProfile(profile),
 		rs.WithParentResourceID(parent),
-		rs.WithAnnotation(&v2.RawId{Id: resourceID}),
+		rs.WithAnnotation(
+			&v2.ChildResourceType{ResourceTypeId: schemaResourceType.Id},
+			&v2.RawId{Id: resourceID},
+		),
 	)
 }
 
@@ -474,7 +478,13 @@ func catalogGrantsAreTrustworthy(
 		}
 	}
 
-	return holdsCatalogManage(ctx, client, trust, annos)
+	manages, err := holdsPrivilegeOn(ctx, client, trust, databricks.SecurableCatalog, trust.fullName,
+		managePrivilege, []string{errorCodeCatalogGone}, annos)
+	if errors.Is(err, errSecurableGone) {
+		return false, errCatalogGone
+	}
+
+	return manages, err
 }
 
 func catalogAdminWorkspaceID(owner string) (string, bool) {
@@ -493,61 +503,6 @@ func catalogAdminWorkspaceID(owner string) (string, bool) {
 	return id, true
 }
 
-// holdsCatalogManage asks the permissions endpoint for the connector's own
-// effective privileges on the catalog. The principal filter is effective, not
-// literal: a row can be the MANAGE a group holds and this principal inherits, so
-// only the privileges are read and never the principal field. The read bypasses
-// the cache, because a remembered page would decide this call with the access of
-// an earlier one.
-//
-// ALL_PRIVILEGES is not enough. Databricks leaves MANAGE out of it, so its holder
-// still reads a partial list.
-// https://docs.databricks.com/aws/en/data-governance/unity-catalog/manage-privileges/privileges
-func holdsCatalogManage(
-	ctx context.Context,
-	client *databricks.Client,
-	trust securableTrust,
-	annos *annotations.Annotations,
-) (bool, error) {
-	if trust.ownPrincipal == "" || trust.workspace == "" {
-		return false, nil
-	}
-
-	var manages bool
-	rateLimit, err := client.ForEachUncachedPermissionsPage(ctx, trust.workspace, databricks.SecurableCatalog, trust.fullName, trust.ownPrincipal,
-		func(page []databricks.PrivilegeAssignment) (bool, error) {
-			for _, assignment := range page {
-				if slices.Contains(assignment.Privileges, privilegeManage) {
-					manages = true
-
-					return true, nil
-				}
-			}
-
-			return false, nil
-		})
-	noteRateLimit(annos, rateLimit)
-	if err != nil {
-		if databricks.IsSecurableGone(err, errorCodeCatalogGone) {
-			return false, errCatalogGone
-		}
-		if apiStatus(err) == http.StatusNotFound {
-			return false, refuseUnreadable(
-				fmt.Sprintf("databricks-connector: failed to read the connector's own privileges on catalog %s", trust.fullName), err)
-		}
-		if !deniesTrust(err) {
-			return false, fmt.Errorf("databricks-connector: failed to read the connector's own privileges on catalog %s: %w", trust.fullName, err)
-		}
-		ctxzap.Extract(ctx).Debug("databricks-connector: the connector's own privileges on the catalog cannot be read, so MANAGE stays unproven",
-			zap.String("catalog", trust.fullName),
-			zap.Error(err))
-
-		return false, nil
-	}
-
-	return manages, nil
-}
-
 func (b *catalogBuilder) Grants(ctx context.Context, resource *v2.Resource, attr rs.SyncOpAttrs) ([]*v2.Grant, *rs.SyncOpResults, error) {
 	return b.syncGrants(ctx, resource, attr)
 }

pkg/connector/catalog_test.go

@@ -415,7 +415,7 @@ func TestCatalogListSkipsBuiltInCatalogs(t *testing.T) {
 	f := &fakeCatalogAccount{
 		attached: []int{1},
 		listed: map[string][]map[string]any{"wsa": {
-			catalogEntry(systemCatalog, "someone"),
+			catalogEntry(systemCatalog, "System user"),
 			{"name": "samples", "metastore_id": "ms-1", "owner": "System user", "catalog_type": "SYSTEM_CATALOG"},
 			catalogEntry("owned", "app-9"),
 		}},

pkg/connector/connector.go

@@ -44,6 +44,7 @@ func (d *Databricks) ResourceSyncers(_ context.Context) []connectorbuilder.Resou
 		newRoleBuilder(d.client),
 		newMetastoreBuilder(d.client, d.clientID, workspaces, principals),
 		newCatalogBuilder(d.client, d.clientID, workspaces, principals),
+		newSchemaBuilder(d.client, d.clientID, workspaces, principals),
 	}
 }
 
@@ -75,7 +76,7 @@ func (d *Databricks) Metadata(_ context.Context) (*v2.ConnectorMetadata, error)
 	return &v2.ConnectorMetadata{
 		DisplayName: "Databricks",
 		Description: "Connector syncing Databricks workspaces, users, groups, service principals and roles to Baton, " +
-			"along with Unity Catalog data assets — metastores and catalogs — and the privileges granted on them",
+			"along with Unity Catalog data assets — metastores, catalogs and schemas — and the privileges granted on them",
 		AccountCreationSchema: &v2.ConnectorAccountCreationSchema{
 			FieldMap: map[string]*v2.ConnectorAccountCreationSchema_Field{
 				"email": {

pkg/connector/connector_test.go

@@ -87,7 +87,7 @@ func TestResourceSyncersAlwaysRegisterTheUnityCatalogBuilders(t *testing.T) {
 	for _, syncer := range d.ResourceSyncers(ctx) {
 		registered = append(registered, syncer.ResourceType(ctx).GetId())
 	}
-	for _, want := range []string{metastoreResourceType.Id, catalogResourceType.Id} {
+	for _, want := range []string{metastoreResourceType.Id, catalogResourceType.Id, schemaResourceType.Id} {
 		if !slices.Contains(registered, want) {
 			t.Errorf("registered syncers = %v, want them to include %s", registered, want)
 		}