[CXH-2723] - Add Unity Catalog table and volume sync - Databricks Connector

README.md

@@ -83,6 +83,9 @@ baton resources
 - Roles
 - Metastores
 - Catalogs (under their metastore)
+- Schemas (under their catalog)
+- Tables (under their schema)
+- Volumes (under their schema)
 
 The connector fetches all resources from the account and every workspace the
 service principal can access. To limit the scope, pass a comma-separated list of
@@ -95,9 +98,9 @@ other metastores and their grants from ConductorOne. Catalogs must also be
 visible through a listed workspace. A list that names no workspace on the
 account is rejected during credential validation, before any sync runs.
 
-Metastores and catalogs are opt-in: each resource type stays off until enabled
-for the connector in ConductorOne. Catalog sync uses the existing OAuth
-credentials and configuration fields.
+Metastores, catalogs, schemas, tables, and volumes are Unity Catalog securables
+and are opt-in: each type stays off until it is enabled for the connector in
+ConductorOne. They use the existing OAuth credentials and configuration fields.
 
 ## Authentication
 
@@ -223,6 +226,8 @@ account
 └── metastore
     └── catalog
         └── schema
+            ├── table
+            └── volume
 ```
 
 A metastore exposes its privileges as entitlements (`CREATE_CATALOG`,
@@ -264,21 +269,26 @@ type) are excluded, regardless of ownership.
 
 ## Unity Catalog provisioning
 
-A metastore or catalog privilege can be granted to and revoked from a user,
-group or service principal. The `owner` entitlement is read-only: granting or
-revoking it fails with an `InvalidArgument` error.
+A privilege on a metastore, catalog, schema, table, or volume can be granted to
+and revoked from a user, group or service principal. The `owner` entitlement is
+read-only: granting or revoking it fails with an `InvalidArgument` error.
 
-If a catalog becomes unreachable, provisioning must distinguish deletion from
+If a securable becomes unreachable, provisioning must distinguish deletion from
 a workspace binding or visibility change. Requests can fail with
 `FailedPrecondition` when deletion cannot be proved.
 
 ## Unity Catalog permissions
 
 The service principal must own the metastore, or be assigned **Admin** directly
 on at least one running workspace attached to it. Admin inherited only through a
-group is not enough. Databricks answers an under-permissioned read with HTTP 200
-and only the rows the caller may see, so the connector reads a metastore only
-through a workspace that passes that check. When no attached workspace does, the
+group is not enough. The connector reads a metastore only through a workspace
+that passes that check. Listing every schema takes ownership of the catalog, or
+`MANAGE`, `USE_SCHEMA`, or `ALL_PRIVILEGES` on that catalog. Listing every table
+or volume needs more: ownership of the metastore, the catalog, or the schema,
+direct Admin when the workspace admins group owns the catalog, or
+`ALL_PRIVILEGES` on the catalog. `MANAGE` and `USE_SCHEMA` are not enough for
+tables or volumes. Databricks answers an under-permissioned read with HTTP 200
+and only the rows the caller may see. The connector refuses that read, so the
 sync fails and ConductorOne keeps the grants it already has.
 
 For a catalog's full grants list, the service principal must own the metastore

baton_capabilities.json

@@ -377,6 +377,82 @@
         ]
       }
     },
+    {
+      "resourceType": {
+        "id": "table",
+        "displayName": "Table",
+        "traits": [
+          "TRAIT_APP"
+        ],
+        "annotations": [
+          {
+            "@type": "type.googleapis.com/c1.connector.v2.OptInRequired"
+          },
+          {
+            "@type": "type.googleapis.com/c1.connector.v2.CapabilityPermissions",
+            "permissions": [
+              {
+                "permission": "unity-catalog"
+              },
+              {
+                "permission": "scim"
+              },
+              {
+                "permission": "access-management"
+              },
+              {
+                "permission": "provisioning"
+              },
+              {
+                "permission": "Account admin"
+              },
+              {
+                "permission": "Workspace admin"
+              },
+              {
+                "permission": "USE_CATALOG"
+              },
+              {
+                "permission": "USE_SCHEMA"
+              }
+            ]
+          }
+        ]
+      },
+      "capabilities": [
+        "CAPABILITY_SYNC",
+        "CAPABILITY_PROVISION"
+      ],
+      "permissions": {
+        "permissions": [
+          {
+            "permission": "unity-catalog"
+          },
+          {
+            "permission": "scim"
+          },
+          {
+            "permission": "access-management"
+          },
+          {
+            "permission": "provisioning"
+          },
+          {
+            "permission": "Account admin"
+          },
+          {
+            "permission": "Workspace admin"
+          },
+          {
+            "permission": "USE_CATALOG"
+          },
+          {
+            "permission": "USE_SCHEMA"
+          }
+        ]
+      },
+      "optInRequired": true
+    },
     {
       "resourceType": {
         "id": "user",
@@ -418,6 +494,85 @@
         ]
       }
     },
+    {
+      "resourceType": {
+        "id": "volume",
+        "displayName": "Volume",
+        "traits": [
+          "TRAIT_APP"
+        ],
+        "annotations": [
+          {
+            "@type": "type.googleapis.com/c1.connector.v2.OptInRequired"
+          },
+          {
+            "@type": "type.googleapis.com/c1.connector.v2.CapabilityPermissions",
+            "permissions": [
+              {
+                "permission": "unity-catalog"
+              },
+              {
+                "permission": "scim"
+              },
+              {
+                "permission": "access-management"
+              },
+              {
+                "permission": "provisioning"
+              },
+              {
+                "permission": "Account admin"
+              },
+              {
+                "permission": "Workspace admin"
+              },
+              {
+                "permission": "USE_CATALOG"
+              },
+              {
+                "permission": "USE_SCHEMA"
+              }
+            ]
+          },
+          {
+            "@type": "type.googleapis.com/c1.connector.v2.SkipEntitlements"
+          }
+        ]
+      },
+      "capabilities": [
+        "CAPABILITY_SYNC",
+        "CAPABILITY_PROVISION"
+      ],
+      "permissions": {
+        "permissions": [
+          {
+            "permission": "unity-catalog"
+          },
+          {
+            "permission": "scim"
+          },
+          {
+            "permission": "access-management"
+          },
+          {
+            "permission": "provisioning"
+          },
+          {
+            "permission": "Account admin"
+          },
+          {
+            "permission": "Workspace admin"
+          },
+          {
+            "permission": "USE_CATALOG"
+          },
+          {
+            "permission": "USE_SCHEMA"
+          }
+        ]
+      },
+      "optInRequired": true
+    },
     {
       "resourceType": {
         "id": "workspace",

docs/connector.mdx

@@ -17,7 +17,9 @@ sidebarTitle: "Databricks"
 | Roles | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> |
 | Schemas | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> |
 | Service principals | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> |
+| Tables | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> |
 | Users | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> |
+| Volumes | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> |
 | Workspaces | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> | <Icon icon="square-check" iconType="solid"  color="#c937ae"/> |
 
 The Databricks connector supports [automatic account provisioning and deprovisioning](/product/admin/account-provisioning).
@@ -30,25 +32,25 @@ Revoking the workspace **member** entitlement removes the principal's entire wor
 
 [This connector syncs non-human identities](/product/admin/nhi) and displays them on the **Identities overview** dashboard.
 
-## Metastore, catalog and schema access in C1
+## Metastore, catalog, schema, table and volume access in C1
 
 The connector syncs Databricks [Unity Catalog](https://docs.databricks.com/aws/en/data-governance/unity-catalog/manage-privileges/privileges)
-metastores, the catalogs in them, and the schemas in those catalogs as C1 resources. Each privilege Databricks
+metastores, the catalogs in them, the schemas in those catalogs, and the tables and volumes in those schemas as C1 resources. Each privilege Databricks
 records on one of them (`CREATE_CATALOG` on a metastore, `USE_CATALOG` or
-`SELECT` on a catalog, `USE_SCHEMA` on a schema, and so on) becomes an entitlement in C1, so you can
+`SELECT` on a catalog, `USE_SCHEMA` on a schema, `SELECT` on a table, `READ_VOLUME` on a volume, and so on) becomes an entitlement in C1, so you can
 review who holds it, request it, and revoke it for users, groups and service
 principals. A grant held by a group is expandable so C1 can resolve the
 group's members as holders. Ownership is shown as a read-only `owner`
-entitlement — it cannot be granted or revoked through C1.
+entitlement — it cannot be granted or revoked through C1. Views, materialized views, metric views and streaming tables are tables; `table_type` picks which privileges can be requested. `ALL_PRIVILEGES` on a view, and `MODIFY` on a streaming table, are shown when they already exist and cannot be requested.
 
 <Note>
-The Metastores, Catalogs and Schemas resource types are turned off by default. Turn on the ones you want from the connector's configuration page in C1. Catalogs appear under their metastore and schemas under their catalog, and all of them use the same OAuth credentials; no additional configuration fields are needed.
+The Metastores, Catalogs, Schemas, Tables and Volumes resource types are turned off by default. Turn on the ones you want from the connector's configuration page in C1. Each appears under its parent securable, and all of them use the same OAuth credentials; no additional configuration fields are needed.
 </Note>
 
 <Note>
 **Notes on Unity Catalog access in C1:**
 
-- Ownership is shown but cannot be provisioned. An owner that is not a Databricks account identity, such as a system account or a workspace admins group, is not shown at all.
+- Ownership is shown but cannot be provisioned. An owner that is not an account identity, such as `System user`, is not shown. A workspace admins group is shown when it resolves as an account group.
 - **MANAGE** and **ALL_PRIVILEGES** are not offered on a metastore, because Databricks rejects both at that level.
 - A catalog is synced when any running workspace in scope attached to its metastore lists it, and the same catalog name is kept once per metastore. A catalog that only excluded workspaces can reach is not synced.
 - The built-in `system` catalog and catalogs Databricks identifies as `SYSTEM_CATALOG` are excluded regardless of ownership. This includes `samples` when Databricks reports that type.
@@ -214,6 +216,8 @@ Workspace **Admin** does not list every schema, except when the catalog is owned
 A catalog whose schemas cannot all be listed fails the sync. `system` and `samples` are not a special case of that listing: the catalog sync leaves them out, when their privilege list cannot be proven complete, before their schemas are read.
 </Warning>
 
+Listing every table or volume needs more than **USE SCHEMA** or **MANAGE**. The service principal must own the metastore, the catalog, or the schema, be a direct Admin when the workspace admins group owns the catalog, or hold **ALL PRIVILEGES** on the catalog. Otherwise the sync fails.
+
 **Done.** Here's the set of credentials you'll need when setting up the connector:
 
 - Account ID

docs/docs-info.md

@@ -6,15 +6,15 @@ While developing the connector, please fill out this form. This information is n
 
    > The connector syncs the Databricks account, workspaces, users, groups, service principals, and roles.
    >
-   > It also syncs Unity Catalog metastores and their catalogs. Both resource types are opt-in — each stays off until enabled per resource type in ConductorOne.
+   > It also syncs Unity Catalog metastores, catalogs, schemas, tables, and volumes. Those resource types are opt-in — each stays off until enabled per resource type in ConductorOne.
 
 2. Can the connector provision any resources? If so, which ones?
 
    > Yes:
    >
    > - **User accounts**: create and delete account users.
    > - **Entitlements**: grant and revoke role and membership assignments on accounts, workspaces, groups, service principals, and roles.
-   > - **Unity Catalog privileges**: grant and revoke a metastore or catalog privilege for a user, group or service principal. The `owner` entitlement is read-only and cannot be granted or revoked.
+   > - **Unity Catalog privileges**: grant and revoke a privilege on a metastore, catalog, schema, table, or volume for a user, group or service principal. The `owner` entitlement is read-only and cannot be granted or revoked.
 
 ## Connector credentials
 
@@ -57,12 +57,13 @@ While developing the connector, please fill out this form. This information is n
 
 - **Direct grants only.** Databricks inherits privileges downward. The connector reports the assignments stored on the object. The inheritance endpoint does not return principal IDs.
 - **Full grants visibility is required.** Metastore ownership or direct workspace Admin permits metastore reads. Catalog reads require one of the catalog permissions listed above; workspace Admin applies only when that workspace's admins group owns the catalog. Admin inherited through a group does not count. The sync fails instead of reporting a partial list, so ConductorOne keeps the grants it already has.
-- **Ownership is read-only.** It is shown and cannot be granted or revoked. An owner that is not an account identity (`System user`, `_workspace_admins_…`) is not shown.
+- **A partial list fails the sync.** A privilege read on a catalog, schema, table, or volume is complete when the service principal owns the metastore, owns the object or a parent, or holds `MANAGE`. `ALL_PRIVILEGES` does not include `MANAGE`. A schema listing is also complete with `USE_SCHEMA` or `ALL_PRIVILEGES`, or when the catalog is owned by the workspace admins group and the service principal is a direct Admin. A table or volume listing is not: it needs ownership of the metastore, catalog, or schema, that same Admin standing, or `ALL_PRIVILEGES` on the catalog. Without that standing the sync fails naming the object, and ConductorOne keeps what it already has. `system` and `samples` (any `SYSTEM_CATALOG`) are left out of the catalog sync when their privilege list cannot be proven complete. While the service principal owns the metastore, both are synced, including their schemas and tables.
+- **Ownership is read-only.** It is shown and cannot be granted or revoked. An owner that does not resolve to an account identity, such as `System user`, is not shown. A workspace admins group is shown when it resolves as an account group.
 - **No `MANAGE` or `ALL_PRIVILEGES` on a metastore.** Databricks rejects both, so they are not offered.
 - **Read through one running workspace.** Privileges are not served on the account plane. The connector reads each metastore through the first running workspace in scope where it can see every privilege. With no running workspace in scope, the metastore is not synced and ConductorOne removes it. A workspace that is still provisioning fails the sync until it is `RUNNING`. An unreadable assignment list fails the sync and ConductorOne keeps the metastore.
 - **`workspaces` scopes metastores.** An allowlist that names no workspace on the account fails at credential validation, before any sync. A name that is also in `databricks-exclude-workspaces` is reported separately from a name the account does not have.
 - **Catalogs are opt-in.** The resource type is off until it is turned on from the connector's configuration page in C1.
 - **Catalog discovery visits every attached workspace in scope.** The connector asks each running workspace attached to the metastore, including for `ISOLATED` catalogs bound to only some workspaces. Each catalog name is kept once per metastore. A catalog bound only to workspaces the sync excludes is not listed, and ConductorOne removes it with its grants.
 - **System catalogs are excluded regardless of ownership.** The built-in `system` catalog and any catalog with `CatalogType == "SYSTEM_CATALOG"` are excluded, including `samples` when it has that type.
 - **Seeing a catalog needs more than `MANAGE`.** The list holds the catalogs the service principal owns or holds `USE_CATALOG` on, or every catalog when it is the metastore admin. `MANAGE` alone reads a catalog's privileges but does not make it appear.
-- **Not synced yet:** tables, volumes, functions, registered models, external locations, storage credentials, connections, shares, recipients, and tags.
+- **Not synced yet:** functions, registered models, external locations, storage credentials, connections, shares, recipients, and tags.

pkg/connector/catalog.go

@@ -122,7 +122,6 @@ var errCatalogGone = errors.New("catalog no longer exists")
 
 type catalogBuilder struct {
 	securableOps
-	// workspaces is the configured allowlist, empty when none was given.
 	workspaces map[string]struct{}
 }
 

pkg/connector/connector.go

@@ -45,6 +45,8 @@ func (d *Databricks) ResourceSyncers(_ context.Context) []connectorbuilder.Resou
 		newMetastoreBuilder(d.client, d.clientID, workspaces, principals),
 		newCatalogBuilder(d.client, d.clientID, workspaces, principals),
 		newSchemaBuilder(d.client, d.clientID, workspaces, principals),
+		newTableBuilder(d.client, d.clientID, workspaces, principals),
+		newVolumeBuilder(d.client, d.clientID, workspaces, principals),
 	}
 }
 
@@ -76,7 +78,7 @@ func (d *Databricks) Metadata(_ context.Context) (*v2.ConnectorMetadata, error)
 	return &v2.ConnectorMetadata{
 		DisplayName: "Databricks",
 		Description: "Connector syncing Databricks workspaces, users, groups, service principals and roles to Baton, " +
-			"along with Unity Catalog data assets — metastores, catalogs and schemas — and the privileges granted on them",
+			"along with Unity Catalog data assets — metastores, catalogs, schemas, tables and volumes — and the privileges granted on them",
 		AccountCreationSchema: &v2.ConnectorAccountCreationSchema{
 			FieldMap: map[string]*v2.ConnectorAccountCreationSchema_Field{
 				"email": {

pkg/connector/connector_test.go

@@ -87,7 +87,10 @@ func TestResourceSyncersAlwaysRegisterTheUnityCatalogBuilders(t *testing.T) {
 	for _, syncer := range d.ResourceSyncers(ctx) {
 		registered = append(registered, syncer.ResourceType(ctx).GetId())
 	}
-	for _, want := range []string{metastoreResourceType.Id, catalogResourceType.Id, schemaResourceType.Id} {
+	for _, want := range []string{
+		metastoreResourceType.Id, catalogResourceType.Id, schemaResourceType.Id,
+		tableResourceType.Id, volumeResourceType.Id,
+	} {
 		if !slices.Contains(registered, want) {
 			t.Errorf("registered syncers = %v, want them to include %s", registered, want)
 		}

pkg/connector/metastore.go

@@ -26,7 +26,6 @@ var _ connectorbuilder.StaticEntitlementSyncerV2 = (*metastoreBuilder)(nil)
 
 type metastoreBuilder struct {
 	securableOps
-	// workspaces is the configured allowlist, empty when none was given.
 	workspaces map[string]struct{}
 }