Rename Dataproc Spark Connect to Managed Spark Connect

.env.example

@@ -1,4 +1,4 @@
-# Google Cloud Configuration for Dataproc Spark Connect Integration Tests
+# Google Cloud Configuration for Managed Spark Connect Integration Tests
 # Copy this file to .env and fill in your actual values
 
 # ============================================================================
@@ -8,7 +8,7 @@
 # Your Google Cloud Project ID
 GOOGLE_CLOUD_PROJECT="your-project-id"
 
-# Google Cloud Region where Dataproc sessions will be created
+# Google Cloud Region where Managed Spark sessions will be created
 GOOGLE_CLOUD_REGION="us-central1"
 
 # Path to service account key file (if using SERVICE_ACCOUNT auth)
@@ -19,35 +19,35 @@ GOOGLE_APPLICATION_CREDENTIALS="/path/to/your/service-account-key.json"
 # ============================================================================
 
 # Authentication type (SERVICE_ACCOUNT or END_USER_CREDENTIALS). If not set, API default is used.
-# DATAPROC_SPARK_CONNECT_AUTH_TYPE="SERVICE_ACCOUNT"
-# DATAPROC_SPARK_CONNECT_AUTH_TYPE="END_USER_CREDENTIALS"
+# MANAGED_SPARK_CONNECT_AUTH_TYPE="SERVICE_ACCOUNT"
+# MANAGED_SPARK_CONNECT_AUTH_TYPE="END_USER_CREDENTIALS"
 
 # Service account email for workload authentication (optional)
-# DATAPROC_SPARK_CONNECT_SERVICE_ACCOUNT="your-service-account@your-project.iam.gserviceaccount.com"
+# MANAGED_SPARK_CONNECT_SERVICE_ACCOUNT="your-service-account@your-project.iam.gserviceaccount.com"
 
 # ============================================================================
 # SESSION CONFIGURATION
 # ============================================================================
 
 # Session timeout in seconds (how long session stays active)
-# DATAPROC_SPARK_CONNECT_TTL_SECONDS="3600"
+# MANAGED_SPARK_CONNECT_TTL_SECONDS="3600"
 
 # Session idle timeout in seconds (how long session stays active when idle)
-# DATAPROC_SPARK_CONNECT_IDLE_TTL_SECONDS="900"
+# MANAGED_SPARK_CONNECT_IDLE_TTL_SECONDS="900"
 
 # Automatically terminate session when Python process exits (true/false)
-# DATAPROC_SPARK_CONNECT_SESSION_TERMINATE_AT_EXIT="false"
+# MANAGED_SPARK_CONNECT_SESSION_TERMINATE_AT_EXIT="false"
 
 # Custom file path for storing active session information
-# DATAPROC_SPARK_CONNECT_ACTIVE_SESSION_FILE_PATH="/tmp/dataproc_spark_connect_session"
+# MANAGED_SPARK_CONNECT_ACTIVE_SESSION_FILE_PATH="/tmp/managed_spark_connect_session"
 
 # ============================================================================
 # DATA SOURCE CONFIGURATION
 # ============================================================================
 
 # Default data source for Spark SQL (currently only supports "bigquery")
-# Only available for Dataproc runtime version 2.3
-# DATAPROC_SPARK_CONNECT_DEFAULT_DATASOURCE="bigquery"
+# Only available for Managed Spark runtime version 2.3
+# MANAGED_SPARK_CONNECT_DEFAULT_DATASOURCE="bigquery"
 
 # ============================================================================
 # ADVANCED CONFIGURATION
@@ -56,8 +56,7 @@ GOOGLE_APPLICATION_CREDENTIALS="/path/to/your/service-account-key.json"
 # Custom Dataproc API endpoint (uncomment if needed)
 # GOOGLE_CLOUD_DATAPROC_API_ENDPOINT="your-region-dataproc.googleapis.com"
 
-# Subnet URI for Dataproc Spark Connect (full resource name format)
+# Subnet URI for Managed Spark Connect (full resource name format)
 # Example: projects/your-project-id/regions/us-central1/subnetworks/your-subnet-name
-# DATAPROC_SPARK_CONNECT_SUBNET="projects/your-project-id/regions/us-central1/subnetworks/your-subnet-name"
-
+# MANAGED_SPARK_CONNECT_SUBNET="projects/your-project-id/regions/us-central1/subnetworks/your-subnet-name"
 

.github/workflows/integration-tests.yaml

@@ -17,7 +17,7 @@
 # Required GitHub Secrets:
 #   - GCP_SA_KEY: Service account JSON key (project_id and client_email extracted automatically)
 #   - GCP_REGION: Google Cloud Region (optional, defaults to us-central1)
-#   - GCP_SUBNET: Dataproc subnet URI
+#   - GCP_SUBNET: Managed Spark subnet URI
 #
 # See INTEGRATION_TESTS.md for setup instructions.
 
@@ -27,6 +27,9 @@ on:
     branches: [ main ]
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   integration-test:
     name: Run integration tests
@@ -37,15 +40,17 @@ jobs:
 
     steps:
     - name: Checkout code
-      uses: actions/checkout@v4
+      uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+      with:
+        persist-credentials: false
 
     - name: Setup Python
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
       with:
         python-version: "3.12"
 
     - name: Cache pip dependencies
-      uses: actions/cache@v4
+      uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
       with:
         path: ~/.cache/pip
         key: ${{ runner.os }}-pip-integration-${{ hashFiles('requirements-dev.txt', 'requirements-test.txt') }}
@@ -59,22 +64,33 @@ jobs:
         pip install -r requirements-test.txt
 
     - name: Authenticate to Google Cloud
-      uses: google-github-actions/auth@v2
+      uses: google-github-actions/auth@c200f3691d83b41bf9bbd8638997a462592937ed # v2
       with:
         credentials_json: ${{ secrets.GCP_SA_KEY }}
 
     - name: Set up Cloud SDK
-      uses: google-github-actions/setup-gcloud@v2
+      uses: google-github-actions/setup-gcloud@e427ad8a34f8676edf47cf7d7925499adf3eb74f # v2
+
+    - name: Extract service account details
+      env:
+        GCP_SA_KEY_JSON: ${{ secrets.GCP_SA_KEY }}
+      run: |
+        SA_EMAIL=$(echo "$GCP_SA_KEY_JSON" | jq -r '.client_email')
+        PROJECT_ID=$(echo "$GCP_SA_KEY_JSON" | jq -r '.project_id')
+        echo "::add-mask::$SA_EMAIL"
+        echo "::add-mask::$PROJECT_ID"
+        echo "SA_EMAIL=$SA_EMAIL" >> "$GITHUB_ENV"
+        echo "PROJECT_ID=$PROJECT_ID" >> "$GITHUB_ENV"
 
     - name: Run integration tests
       env:
         CI: "true"
-        # Extract from service account JSON automatically
-        GOOGLE_CLOUD_PROJECT: ${{ fromJson(secrets.GCP_SA_KEY).project_id }}
-        DATAPROC_SPARK_CONNECT_SERVICE_ACCOUNT: ${{ fromJson(secrets.GCP_SA_KEY).client_email }}
+        # Extracted from service account JSON in the previous step
+        GOOGLE_CLOUD_PROJECT: ${{ env.PROJECT_ID }}
+        MANAGED_SPARK_CONNECT_SERVICE_ACCOUNT: ${{ env.SA_EMAIL }}
         # Infrastructure-specific secrets
         GOOGLE_CLOUD_REGION: ${{ secrets.GCP_REGION || 'us-central1' }}
-        DATAPROC_SPARK_CONNECT_SUBNET: ${{ secrets.GCP_SUBNET }}
-        DATAPROC_SPARK_CONNECT_AUTH_TYPE: "SERVICE_ACCOUNT"
+        MANAGED_SPARK_CONNECT_SUBNET: ${{ secrets.GCP_SUBNET }}
+        MANAGED_SPARK_CONNECT_AUTH_TYPE: "SERVICE_ACCOUNT"
       run: |
         python -m pytest tests/integration/ -v --tb=short -x

DEVELOPING.md

@@ -35,7 +35,7 @@ configuration details on the command line. For example:
 env \
   GOOGLE_CLOUD_PROJECT='project-id' \
   GOOGLE_CLOUD_REGION='us-central1' \
-  DATAPROC_SPARK_CONNECT_SUBNET='subnet-id' \
+  MANAGED_SPARK_CONNECT_SUBNET='subnet-id' \
   pytest --tb=auto -v
 ```
 
@@ -70,7 +70,7 @@ use. This will be set automatically if you set it to `auto`. For example:
 env \
   GOOGLE_CLOUD_PROJECT='project-id' \
   GOOGLE_CLOUD_REGION='us-central1' \
-  DATAPROC_SPARK_CONNECT_SUBNET='subnet-id' \
-  DATAPROC_SPARK_CONNECT_SERVICE_ACCOUNT='service@account.test' \
+  MANAGED_SPARK_CONNECT_SUBNET='subnet-id' \
+  MANAGED_SPARK_CONNECT_SERVICE_ACCOUNT='service@account.test' \
   pytest -n auto --tb=auto -v
 ```

README.md

@@ -1,26 +1,26 @@
-# Dataproc Spark Connect Client
+# Managed Spark Connect Client
 
 A wrapper of the Apache [Spark Connect](https://spark.apache.org/spark-connect/)
 client with additional functionalities that allow applications to communicate
-with a remote Dataproc Spark Session using the Spark Connect protocol without
+with a remote Managed Spark Session using the Spark Connect protocol without
 requiring additional steps.
 
 ## Install
 
 ```sh
-pip install dataproc_spark_connect
+pip install google-cloud-managed-spark-connect
 ```
 
 ## Uninstall
 
 ```sh
-pip uninstall dataproc_spark_connect
+pip uninstall google-cloud-managed-spark-connect
 ```
 
 ## Setup
 
 This client requires permissions to
-manage [Dataproc Sessions and Session Templates](https://cloud.google.com/dataproc-serverless/docs/concepts/iam).
+manage [Managed Spark Sessions and Session Templates](https://cloud.google.com/dataproc-serverless/docs/concepts/iam).
 
 If you are running the client outside of Google Cloud, you need to provide
 authentication credentials. Set the `GOOGLE_APPLICATION_CREDENTIALS` environment
@@ -36,42 +36,42 @@ in your code using the builder API:
 
 ## Usage
 
-1. Install the latest version of Dataproc Spark Connect:
+1. Install the latest version of Managed Spark Connect:
 
    ```sh
-   pip install -U dataproc-spark-connect
+   pip install -U google-cloud-managed-spark-connect
    ```
 
 2. Add the required imports into your PySpark application or notebook and start
    a Spark session using the fluent API:
 
    ```python
-   from google.cloud.dataproc_spark_connect import DataprocSparkSession
-   spark = DataprocSparkSession.builder.getOrCreate()
+   from google.cloud.managed_spark_connect import ManagedSparkSession
+   spark = ManagedSparkSession.builder.getOrCreate()
    ```
 
 3. You can configure Spark properties using the `.config()` method:
 
    ```python
-   from google.cloud.dataproc_spark_connect import DataprocSparkSession
-   spark = DataprocSparkSession.builder.config('spark.executor.memory', '4g').config('spark.executor.cores', '2').getOrCreate()
+   from google.cloud.managed_spark_connect import ManagedSparkSession
+   spark = ManagedSparkSession.builder.config('spark.executor.memory', '4g').config('spark.executor.cores', '2').getOrCreate()
    ```
 
 4. For advanced configuration, you can use the `Session` class to customize
    settings like subnetwork or other environment configurations:
 
    ```python
-   from google.cloud.dataproc_spark_connect import DataprocSparkSession
+   from google.cloud.managed_spark_connect import ManagedSparkSession
    from google.cloud.dataproc_v1 import Session
    session_config = Session()
    session_config.environment_config.execution_config.subnetwork_uri = '<subnet>'
    session_config.runtime_config.version = '3.0'
-   spark = DataprocSparkSession.builder.projectId('my-project').location('us-central1').dataprocSessionConfig(session_config).getOrCreate()
+   spark = ManagedSparkSession.builder.projectId('my-project').location('us-central1').dataprocSessionConfig(session_config).getOrCreate()
    ```
 
 ### Builder Configuration
 
-The `DataprocSparkSession.builder` provides a fluent API to configure the session. Below is a list of available methods:
+The `ManagedSparkSession.builder` provides a fluent API to configure the session. Below is a list of available methods:
 
 | Method | Description |
 |--------|-------------|
@@ -83,9 +83,9 @@ The `DataprocSparkSession.builder` provides a fluent API to configure the sessio
 | `labels(labels)` | Adds multiple labels to the session. |
 | `location(location)` | Sets the Google Cloud region. |
 | `projectId(project_id)` | Sets the Google Cloud project ID. |
-| `runtimeVersion(version)` | Sets the Dataproc runtime version (e.g., "3.0"). |
+| `runtimeVersion(version)` | Sets the Managed Spark runtime version (e.g., "3.0"). |
 | `serviceAccount(account)` | Sets the service account for the session. |
-| `sessionTemplate(template)` | Sets the session template to use. |
+| `sessionTemplate(profile)` | Sets the Session Template to use. |
 | `subnetwork(subnet)` | Sets the subnetwork URI for the session. |
 | `ttl(duration)` | Sets the time-to-live (TTL) for the session using a `datetime.timedelta` object. |
 
@@ -98,19 +98,19 @@ To create or connect to a named session:
 1. Create a session with a custom ID in your first notebook:
 
    ```python
-   from google.cloud.dataproc_spark_connect import DataprocSparkSession
+   from google.cloud.managed_spark_connect import ManagedSparkSession
    session_id = 'my-ml-pipeline-session'
-   spark = DataprocSparkSession.builder.dataprocSessionId(session_id).getOrCreate()
+   spark = ManagedSparkSession.builder.dataprocSessionId(session_id).getOrCreate()
    df = spark.createDataFrame([(1, 'data')], ['id', 'value'])
    df.show()
    ```
 
 2. Reuse the same session in another notebook by specifying the same session ID:
 
    ```python
-   from google.cloud.dataproc_spark_connect import DataprocSparkSession
+   from google.cloud.managed_spark_connect import ManagedSparkSession
    session_id = 'my-ml-pipeline-session'
-   spark = DataprocSparkSession.builder.dataprocSessionId(session_id).getOrCreate()
+   spark = ManagedSparkSession.builder.dataprocSessionId(session_id).getOrCreate()
    df = spark.createDataFrame([(2, 'more-data')], ['id', 'value'])
    df.show()
    ```
@@ -127,7 +127,7 @@ The package supports the [sparksql-magic](https://github.com/cryeo/sparksql-magi
 
 **Installation**: To use magic commands, install the required dependencies manually:
 ```bash
-pip install dataproc-spark-connect
+pip install google-cloud-managed-spark-connect
 pip install IPython sparksql-magic
 ```
 
@@ -163,11 +163,58 @@ Available options:
 
 See [sparksql-magic](https://github.com/cryeo/sparksql-magic) for more examples.
 
-**Note**: Magic commands are optional. If you only need basic DataprocSparkSession functionality without Jupyter magic support, install only the base package:
+**Note**: Magic commands are optional. If you only need basic ManagedSparkSession functionality without Jupyter magic support, install only the base package:
 ```bash
+pip install google-cloud-managed-spark-connect
+```
+
+## Migrating from dataproc-spark-connect
+
+The `dataproc-spark-connect` package has been renamed to `google-cloud-managed-spark-connect`. This is a breaking change with no compatibility shims — you need to update your code in the following places when you switch to the new package.
+
+### 1. Update the package you install
+
+```sh
+# Before
 pip install dataproc-spark-connect
+
+# After
+pip install google-cloud-managed-spark-connect
+```
+
+### 2. Update your imports and session class
+
+`google.cloud.dataproc_spark_connect` is now `google.cloud.managed_spark_connect`, and `DataprocSparkSession` is now `ManagedSparkSession`:
+
+```python
+# Before
+from google.cloud.dataproc_spark_connect import DataprocSparkSession
+spark = DataprocSparkSession.builder.getOrCreate()
+
+# After
+from google.cloud.managed_spark_connect import ManagedSparkSession
+spark = ManagedSparkSession.builder.getOrCreate()
 ```
 
+If you use the Jupyter magic commands, `google.cloud.dataproc_magics` is now `google.cloud.managed_spark_magics` and `DataprocMagics` is now `ManagedSparkMagics` (the `%dpip` magic itself is unchanged).
+
+### 3. Rename any `DATAPROC_SPARK_CONNECT_*` environment variables
+
+If you set any of the library's own environment variables (as opposed to standard GCP ones like `GOOGLE_CLOUD_PROJECT`), rename the `DATAPROC_SPARK_CONNECT_` prefix to `MANAGED_SPARK_CONNECT_`:
+
+| Before | After |
+|--------|-------|
+| `DATAPROC_SPARK_CONNECT_SERVICE_ACCOUNT` | `MANAGED_SPARK_CONNECT_SERVICE_ACCOUNT` |
+| `DATAPROC_SPARK_CONNECT_SUBNET` | `MANAGED_SPARK_CONNECT_SUBNET` |
+| `DATAPROC_SPARK_CONNECT_AUTH_TYPE` | `MANAGED_SPARK_CONNECT_AUTH_TYPE` |
+| `DATAPROC_SPARK_CONNECT_TTL_SECONDS` | `MANAGED_SPARK_CONNECT_TTL_SECONDS` |
+| `DATAPROC_SPARK_CONNECT_IDLE_TTL_SECONDS` | `MANAGED_SPARK_CONNECT_IDLE_TTL_SECONDS` |
+| `DATAPROC_SPARK_CONNECT_SESSION_TERMINATE_AT_EXIT` | `MANAGED_SPARK_CONNECT_SESSION_TERMINATE_AT_EXIT` |
+| `DATAPROC_SPARK_CONNECT_DEFAULT_DATASOURCE` | `MANAGED_SPARK_CONNECT_DEFAULT_DATASOURCE` |
+| `DATAPROC_SPARK_CONNECT_ACTIVE_SESSION_FILE_PATH` | `MANAGED_SPARK_CONNECT_ACTIVE_SESSION_FILE_PATH` |
+
+Note that `GOOGLE_CLOUD_DATAPROC_API_ENDPOINT` and other variables naming the actual Dataproc API (not this library's own config) are unchanged.
+
 ## Developing
 
 For development instructions see [guide](DEVELOPING.md).

cloudbuild/cloudbuild.yaml

@@ -3,9 +3,9 @@ steps:
   # distribution artifacts.
   - name: 'gcr.io/cloud-builders/docker'
     id: 'build-container-image'
-    args: ['build', '--tag=gcr.io/${PROJECT_ID}/dataproc-spark-connect/dataproc-spark-connect-presubmit:${BUILD_ID}', -f, 'cloudbuild/Dockerfile', '.']
+    args: ['build', '--tag=gcr.io/${PROJECT_ID}/managed-spark-connect/managed-spark-connect-presubmit:${BUILD_ID}', -f, 'cloudbuild/Dockerfile', '.']
   # Run all unit tests
-  - name: 'gcr.io/${PROJECT_ID}/dataproc-spark-connect/dataproc-spark-connect-presubmit:${BUILD_ID}'
+  - name: 'gcr.io/${PROJECT_ID}/managed-spark-connect/managed-spark-connect-presubmit:${BUILD_ID}'
     id: 'run-unit-tests'
     waitFor: ['build-container-image']
     entrypoint: 'pytest'