From 81c78b38b261e67e8a736e926a49c9b07f591760 Mon Sep 17 00:00:00 2001 From: "@rugpanov" Date: Thu, 3 Sep 2026 14:01:35 +0200 Subject: [PATCH 1/3] feat(python-setup): report manual-setup opt-out telemetry *Why* The manual opt-out added in #2158 (databricks.python.environmentSetup: manual, plus the "Use manual setup" button on the E_FETCH failure) is only observable today via the generic commandExecution event, and opting out by editing the setting directly is invisible. We can't measure how often users fall back to a manually managed environment, or from where. *What* - New python_env.manual_setup.optout event, recorded when the opt-out command write succeeds, with scope (workspace|global) and source (error_popup| command_palette). The E_FETCH popup tags its invocation as error_popup; palette invocations read as command_palette. - Add setupMode (uv | pip | fallback-pip) to the once-per-session python_env.setup.detected event so opt-out prevalence is measurable per session. Derived in one place (resolveSetupMode, beside isUvSetupSuitable): manual => fallback-pip; auto splits by uv-suitability into uv / pip. The exact manager stays in primaryManager. Categorical data only; no paths, package names, or PII. *Verification* - yarn build (tsc) and yarn test:lint clean. - yarn test:unit green (1061 passing). Added unit tests: resolveSetupMode cases; recordManualSetupOptout scope/source; emitDetection setupMode for uv / pip / fallback-pip; and the popup passing {source: "error_popup"}. Co-authored-by: Isaac --- packages/databricks-vscode/src/extension.ts | 71 ++++++++++++------- .../language/PackageManagerTelemetry.test.ts | 40 ++++++++++- .../src/language/PackageManagerTelemetry.ts | 6 +- .../controllers/pythonSetupDeps.test.ts | 7 +- .../controllers/pythonSetupDeps.ts | 9 ++- .../utils/pythonSetupGate.test.ts | 40 ++++++++++- .../src/python-setup/utils/pythonSetupGate.ts | 27 +++++++ .../src/telemetry/constants.ts | 45 ++++++++++++ .../src/telemetry/packageManagerExtensions.ts | 8 ++- .../telemetry/pythonSetupExtensions.test.ts | 31 ++++++++ .../src/telemetry/pythonSetupExtensions.ts | 25 +++++++ 11 files changed, 273 insertions(+), 36 deletions(-) diff --git a/packages/databricks-vscode/src/extension.ts b/packages/databricks-vscode/src/extension.ts index 3b192da57..c51ba90b0 100644 --- a/packages/databricks-vscode/src/extension.ts +++ b/packages/databricks-vscode/src/extension.ts @@ -76,7 +76,8 @@ import {MsPythonExtensionWrapper} from "./language/MsPythonExtensionWrapper"; import {DatabricksEnvFileManager} from "./file-managers/DatabricksEnvFileManager"; import {getContextMetadata, Telemetry, toUserMetadata} from "./telemetry"; import "./telemetry/commandExtensions"; -import {Events, Metadata} from "./telemetry/constants"; +import "./telemetry/pythonSetupExtensions"; +import {Events, Metadata, PythonSetupOptoutSource} from "./telemetry/constants"; import {EnvironmentDependenciesInstaller} from "./language/EnvironmentDependenciesInstaller"; import {setDbnbCellLimits} from "./language/notebooks/DatabricksNbCellLimits"; import {DbConnectStatusBarButton} from "./language/DbConnectStatusBarButton"; @@ -752,7 +753,8 @@ export async function activate( } return connectionManager.cluster ? "cluster" : "none"; }, - () => connectionManager.state === "CONNECTED" + () => connectionManager.state === "CONNECTED", + () => workspaceConfigs.pythonEnvironmentSetup ); context.subscriptions.push( bundleFileWatcher, @@ -1072,33 +1074,48 @@ export async function activate( // turn automated setup off for this project so an existing environment // is used as-is. Workspace scope keeps it scoped and reversible; if no // folder is open we fall back to Global so the write still lands. - telemetry.registerCommand(USE_MANUAL_SETUP_COMMAND_ID, async () => { - const hasFolder = (workspace.workspaceFolders?.length ?? 0) > 0; - const target = hasFolder - ? ConfigurationTarget.Workspace - : ConfigurationTarget.Global; - try { - await workspaceConfigs.setPythonEnvironmentSetup( - "manual", - target - ); - } catch (e) { - // Don't claim success if the write failed — the user would - // otherwise believe automated setup is off when it is not. - await window.showErrorMessage( - `Could not update databricks.python.environmentSetup: ${ - e instanceof Error ? e.message : String(e) - }` + telemetry.registerCommand( + USE_MANUAL_SETUP_COMMAND_ID, + async (arg?: {source?: PythonSetupOptoutSource}) => { + // The E_FETCH popup passes {source: "error_popup"}; a palette + // invocation passes nothing, so anything else reads as palette. + const source: PythonSetupOptoutSource = + arg?.source === "error_popup" + ? "error_popup" + : "command_palette"; + const hasFolder = (workspace.workspaceFolders?.length ?? 0) > 0; + const target = hasFolder + ? ConfigurationTarget.Workspace + : ConfigurationTarget.Global; + try { + await workspaceConfigs.setPythonEnvironmentSetup( + "manual", + target + ); + } catch (e) { + // Don't claim success if the write failed — the user would + // otherwise believe automated setup is off when it is not. + await window.showErrorMessage( + `Could not update databricks.python.environmentSetup: ${ + e instanceof Error ? e.message : String(e) + }` + ); + return; + } + // Record only after the write lands, so the count reflects real + // opt-outs. scope mirrors the ConfigurationTarget written. + telemetry.recordManualSetupOptout({ + scope: hasFolder ? "workspace" : "global", + source, + }); + // Match the message to the scope actually written: "this project" + // for Workspace, "globally" for the no-folder Global fallback. + const scope = hasFolder ? "for this project" : "globally"; + await window.showInformationMessage( + `Automated Python environment setup is now off ${scope} ("databricks.python.environmentSetup": "manual"). Your existing interpreter will be used as-is.` ); - return; } - // Match the message to the scope actually written: "this project" - // for Workspace, "globally" for the no-folder Global fallback. - const scope = hasFolder ? "for this project" : "globally"; - await window.showInformationMessage( - `Automated Python environment setup is now off ${scope} ("databricks.python.environmentSetup": "manual"). Your existing interpreter will be used as-is.` - ); - }), + ), // E_UV_MISSING's "Install uv" button: run uv's official installer in a // terminal so its output stays visible (same pattern as `az login`). // A modal first makes the remote-install step an explicit, informed diff --git a/packages/databricks-vscode/src/language/PackageManagerTelemetry.test.ts b/packages/databricks-vscode/src/language/PackageManagerTelemetry.test.ts index 5e7589077..4da18a3d0 100644 --- a/packages/databricks-vscode/src/language/PackageManagerTelemetry.test.ts +++ b/packages/databricks-vscode/src/language/PackageManagerTelemetry.test.ts @@ -68,6 +68,7 @@ describe(__filename, () => { projectRoot: string; compute?: "cluster" | "serverless" | "none"; connected?: boolean; + setupMode?: "auto" | "manual"; } ) { return new PackageManagerTelemetry( @@ -75,7 +76,8 @@ describe(__filename, () => { noInterpreter, () => opts.projectRoot, () => opts.compute ?? "none", - () => opts.connected ?? true + () => opts.connected ?? true, + () => opts.setupMode ?? "auto" ); } @@ -102,6 +104,42 @@ describe(__filename, () => { expect(e.props["event.targetCompute"]).to.equal("cluster"); expect(e.props["event.setupTrigger"]).to.equal("explicit_command"); expect(e.props["event.interpreterSource"]).to.equal("unknown"); + // primaryManager is uv, yet a real pip signal (requirements-dev.txt) + // makes the project not uv-suitable, so the flow is pip — setupMode + // tracks the effective flow, not the primary manager. + expect(e.props["event.setupMode"]).to.equal("pip"); + }); + + it("reports setupMode=uv for a clean, auto uv project", async () => { + const {telemetry, events} = makeTelemetry("all"); + const projectRoot = makeProject([["uv.lock", "version = 1\n"]]); + const pmt = makePmt(telemetry, {projectRoot}); // setupMode defaults to auto + + await emit(pmt, "auto_open"); + + expect(events[0].props["event.setupMode"]).to.equal("uv"); + }); + + it("reports setupMode=pip for an auto, non-uv (requirements) project", async () => { + const {telemetry, events} = makeTelemetry("all"); + const projectRoot = makeProject([["requirements.txt", "requests\n"]]); + const pmt = makePmt(telemetry, {projectRoot}); // setupMode defaults to auto + + await emit(pmt, "auto_open"); + + expect(events[0].props["event.primaryManager"]).to.equal("pip"); + expect(events[0].props["event.setupMode"]).to.equal("pip"); + }); + + it("reports setupMode=fallback-pip when the user opted out (manual)", async () => { + const {telemetry, events} = makeTelemetry("all"); + // A clean uv project that would be "uv" on auto — manual still wins. + const projectRoot = makeProject([["uv.lock", "version = 1\n"]]); + const pmt = makePmt(telemetry, {projectRoot, setupMode: "manual"}); + + await emit(pmt, "auto_open"); + + expect(events[0].props["event.setupMode"]).to.equal("fallback-pip"); }); it("deduplicates per (trigger, projectRoot) within a session", async () => { diff --git a/packages/databricks-vscode/src/language/PackageManagerTelemetry.ts b/packages/databricks-vscode/src/language/PackageManagerTelemetry.ts index 07987d250..28f656f52 100644 --- a/packages/databricks-vscode/src/language/PackageManagerTelemetry.ts +++ b/packages/databricks-vscode/src/language/PackageManagerTelemetry.ts @@ -7,6 +7,8 @@ import {MsPythonExtensionWrapper} from "./MsPythonExtensionWrapper"; import {ResolvedEnvironment} from "./MsPythonExtensionApi"; import {detectPackageManagers} from "./packageManagerDetection"; import {collectPackageManagerSignals} from "./packageManagerSignals"; +import {resolveSetupMode} from "../python-setup/utils/pythonSetupGate"; +import type {PythonEnvironmentSetupMode} from "../vscode-objs/WorkspaceConfigs"; export type {SetupTrigger}; @@ -34,7 +36,8 @@ export class PackageManagerTelemetry { private readonly pythonExtension: MsPythonExtensionWrapper, private readonly getProjectRoot: () => string | undefined, private readonly getComputeType: () => ComputeType | "none", - private readonly isConnected: () => boolean + private readonly isConnected: () => boolean, + private readonly getSetupMode: () => PythonEnvironmentSetupMode ) {} /** @@ -80,6 +83,7 @@ export class PackageManagerTelemetry { pythonVersion: this.getPythonMinorVersion(env), targetCompute: this.getComputeType(), trigger, + setupMode: resolveSetupMode(this.getSetupMode(), detection), }); } catch (e) { // Detection is measurement-only and must never disrupt setup. diff --git a/packages/databricks-vscode/src/python-setup/controllers/pythonSetupDeps.test.ts b/packages/databricks-vscode/src/python-setup/controllers/pythonSetupDeps.test.ts index 6dbfc1aa5..530ae33f0 100644 --- a/packages/databricks-vscode/src/python-setup/controllers/pythonSetupDeps.test.ts +++ b/packages/databricks-vscode/src/python-setup/controllers/pythonSetupDeps.test.ts @@ -812,9 +812,11 @@ describe("makePythonSetupDeps showError", () => { it("runs the VS Code command when a command-action button is picked", async () => { const original = commands.executeCommand; const executed: string[] = []; + const executedArgs: unknown[] = []; (commands as unknown as {executeCommand: unknown}).executeCommand = - async (command: string) => { + async (command: string, ...args: unknown[]) => { executed.push(command); + executedArgs.push(args[0]); }; const originalOpen = env.openExternal; const opened: string[] = []; @@ -845,6 +847,9 @@ describe("makePythonSetupDeps showError", () => { expect(executed).to.deep.equal([ "databricks.environment.installUv", ]); + // Command-actions are tagged as coming from the error popup, so a + // command can distinguish a popup click from a palette invocation. + expect(executedArgs).to.deep.equal([{source: "error_popup"}]); expect(opened).to.have.length(0); } finally { (commands as unknown as {executeCommand: unknown}).executeCommand = diff --git a/packages/databricks-vscode/src/python-setup/controllers/pythonSetupDeps.ts b/packages/databricks-vscode/src/python-setup/controllers/pythonSetupDeps.ts index a33de7e82..e9300fd09 100644 --- a/packages/databricks-vscode/src/python-setup/controllers/pythonSetupDeps.ts +++ b/packages/databricks-vscode/src/python-setup/controllers/pythonSetupDeps.ts @@ -323,8 +323,13 @@ export function makePythonSetupDeps( if (chosen.command) { // A command-action (e.g. "Install uv", E_FETCH "Use manual // setup") runs a registered VS Code command instead of - // opening a URL. - await commands.executeCommand(chosen.command); + // opening a URL. Tag the invocation as coming from the error + // popup so the command can tell it apart from a palette call + // (the manual-setup opt-out reads this for telemetry; + // commands that take no args simply ignore it). + await commands.executeCommand(chosen.command, { + source: "error_popup", + }); } else if (chosen.url) { const opened = await openExternal(chosen.url); if (!opened) { diff --git a/packages/databricks-vscode/src/python-setup/utils/pythonSetupGate.test.ts b/packages/databricks-vscode/src/python-setup/utils/pythonSetupGate.test.ts index 0f68d690d..5474c3e97 100644 --- a/packages/databricks-vscode/src/python-setup/utils/pythonSetupGate.test.ts +++ b/packages/databricks-vscode/src/python-setup/utils/pythonSetupGate.test.ts @@ -4,7 +4,7 @@ import { PackageManager, PrimaryManager, } from "../../language/packageManagerDetection"; -import {isUvSetupSuitable} from "./pythonSetupGate"; +import {isUvSetupSuitable, resolveSetupMode} from "./pythonSetupGate"; const det = ( primary: PrimaryManager, @@ -108,3 +108,41 @@ describe("isUvSetupSuitable", () => { ).to.equal(false); }); }); + +// The reported setup flow: manual is decisive (opted out → fallback-pip); +// on auto, uv-suitability splits uv from pip. +describe("resolveSetupMode", () => { + it("reports fallback-pip whenever the setting is manual", () => { + // Manual wins regardless of what the project looks like — even a clean + // uv project, because the user turned automated setup off. + expect(resolveSetupMode("manual", det("uv", ["uv"]))).to.equal( + "fallback-pip" + ); + expect( + resolveSetupMode( + "manual", + det("pip", ["pip"], ["requirements.txt"]) + ) + ).to.equal("fallback-pip"); + }); + + it("reports uv for an auto, uv-suitable project", () => { + expect(resolveSetupMode("auto", det("uv", ["uv"]))).to.equal("uv"); + // Greenfield / packaging-shaped pyproject is uv-suitable too. + expect( + resolveSetupMode("auto", det("pip", ["pip"], ["pyproject.pipOnly"])) + ).to.equal("uv"); + }); + + it("reports pip for an auto project driven by a competing manager", () => { + expect( + resolveSetupMode("auto", det("pip", ["pip"], ["requirements.txt"])) + ).to.equal("pip"); + expect( + resolveSetupMode("auto", det("poetry", ["poetry"], ["poetry.lock"])) + ).to.equal("pip"); + expect( + resolveSetupMode("auto", det("conda", ["conda"], ["conda.prefix"])) + ).to.equal("pip"); + }); +}); diff --git a/packages/databricks-vscode/src/python-setup/utils/pythonSetupGate.ts b/packages/databricks-vscode/src/python-setup/utils/pythonSetupGate.ts index b5fe1a783..fdddf7c26 100644 --- a/packages/databricks-vscode/src/python-setup/utils/pythonSetupGate.ts +++ b/packages/databricks-vscode/src/python-setup/utils/pythonSetupGate.ts @@ -3,6 +3,7 @@ import { PackageManager, PackageManagerDetection, } from "../../language/packageManagerDetection"; +import type {PythonEnvironmentSetupMode} from "../../vscode-objs/WorkspaceConfigs"; /** * Managers whose presence means the project is already committed to a @@ -86,3 +87,29 @@ export function isUvSetupSuitable(detection: SuitabilityDetection): boolean { return !effective.some((m) => COMPETING_MANAGERS.includes(m)); } + +/** + * The environment-setup flow actually in effect for a session, as reported on + * `python_env.setup.detected`. Distinct from the `auto | manual` + * `databricks.python.environmentSetup` setting ({@link PythonEnvironmentSetupMode}): + * `manual` maps to `fallback-pip` (the user opted out; an existing interpreter is + * used as-is), while `auto` splits by uv-suitability into `uv` (uv-native flow) + * or `pip` (legacy/non-uv flow — the exact manager stays in `primaryManager`). + */ +export type PythonEnvSetupMode = "uv" | "pip" | "fallback-pip"; + +/** + * Resolve the reported setup mode from the user's setting and the detected + * project. `manual` is decisive (opted out); on `auto`, uv-suitability decides. + * Kept beside {@link isUvSetupSuitable} so "which flow is in effect" has a single + * source of truth shared by the telemetry emitter. + */ +export function resolveSetupMode( + setting: PythonEnvironmentSetupMode, + detection: SuitabilityDetection +): PythonEnvSetupMode { + if (setting === "manual") { + return "fallback-pip"; + } + return isUvSetupSuitable(detection) ? "uv" : "pip"; +} diff --git a/packages/databricks-vscode/src/telemetry/constants.ts b/packages/databricks-vscode/src/telemetry/constants.ts index 834cdd5f6..b02ce6313 100644 --- a/packages/databricks-vscode/src/telemetry/constants.ts +++ b/packages/databricks-vscode/src/telemetry/constants.ts @@ -31,6 +31,7 @@ export enum Events { PYTHON_ENV_DRIFT = "python_env.drift", PYTHON_ENV_ADOPTION = "python_env.adoption", PYTHON_ENV_DBCONNECT_INSTALL = "python_env.dbconnect_install", + PYTHON_ENV_MANUAL_SETUP_OPTOUT = "python_env.manual_setup.optout", AITOOLS_INSTALL = "aitoolsInstall", AITOOLS_UPDATE = "aitoolsUpdate", AITOOLS_UNINSTALL = "aitoolsUninstall", @@ -103,6 +104,23 @@ import type { } from "../language/packageManagerDetection"; export type {PackageManager, PrimaryManager, InterpreterSource}; +// The effective setup-flow union lives with the uv-suitability gate that derives +// it (the single source of truth), re-exported here so the event schema and the +// resolver can never drift. Type-only, so no runtime dependency/cycle. +import type {PythonEnvSetupMode} from "../python-setup/utils/pythonSetupGate"; +export type {PythonEnvSetupMode}; + +/** + * Which configuration scope the manual opt-out was written to: `workspace` when + * a project folder is open, `global` for the no-folder fallback. + */ +export type PythonSetupOptoutScope = "workspace" | "global"; +/** + * Where the manual opt-out was triggered: the `error_popup` "Use manual setup" + * button on the E_FETCH failure, or the `command_palette`. + */ +export type PythonSetupOptoutSource = "error_popup" | "command_palette"; + /** The compute targeted at the time of detection. */ export type TargetCompute = ComputeType | "none"; /** What triggered a package-manager detection emission. */ @@ -454,6 +472,7 @@ export class EventTypes { hasLockfile: boolean; targetCompute: TargetCompute; setupTrigger: SetupTrigger; + setupMode: PythonEnvSetupMode; }> = { comment: "The Python package/environment manager(s) detected for a project at setup time. " + @@ -488,6 +507,32 @@ export class EventTypes { setupTrigger: { comment: "Which setup touchpoint triggered detection", }, + setupMode: { + comment: + "The environment-setup flow in effect this session: uv (uv-native setup), pip " + + "(legacy/non-uv flow; the exact manager is in primaryManager), or fallback-pip " + + "(user opted out via databricks.python.environmentSetup=manual). The auto|manual " + + "setting is recoverable: fallback-pip <=> manual, uv|pip <=> auto", + }, + }; + [Events.PYTHON_ENV_MANUAL_SETUP_OPTOUT]: EventType<{ + scope: PythonSetupOptoutScope; + source: PythonSetupOptoutSource; + }> = { + comment: + "The user opted out of automated uv-native Python setup by switching " + + "databricks.python.environmentSetup to manual, so an existing interpreter is used " + + "as-is. Recorded only when the setting write succeeds. Categorical data only.", + scope: { + comment: + "Which configuration scope was written: workspace (a project folder is open) or " + + "global (no folder; the fallback so the write still lands)", + }, + source: { + comment: + "Where the opt-out was triggered: error_popup (the 'Use manual setup' button on " + + "the E_FETCH setup failure) or command_palette", + }, }; [Events.PYTHON_ENV_SETUP_ATTEMPT]: EventType<{ packageManager: PrimaryManager; diff --git a/packages/databricks-vscode/src/telemetry/packageManagerExtensions.ts b/packages/databricks-vscode/src/telemetry/packageManagerExtensions.ts index 162e5feca..22f91d515 100644 --- a/packages/databricks-vscode/src/telemetry/packageManagerExtensions.ts +++ b/packages/databricks-vscode/src/telemetry/packageManagerExtensions.ts @@ -1,16 +1,17 @@ import {Events, Telemetry} from "."; -import {TargetCompute, SetupTrigger} from "./constants"; +import {TargetCompute, SetupTrigger, PythonEnvSetupMode} from "./constants"; import {PackageManagerDetection} from "../language/packageManagerDetection"; /** * Context for a package-manager detection that is not part of the detection - * result itself: the interpreter version, the targeted compute, and what - * triggered the emission. + * result itself: the interpreter version, the targeted compute, what triggered + * the emission, and the effective setup flow. */ export interface PackageManagerDetectionContext { pythonVersion?: string; targetCompute: TargetCompute; trigger: SetupTrigger; + setupMode: PythonEnvSetupMode; } declare module "." { @@ -41,6 +42,7 @@ Telemetry.prototype.recordPackageManagerDetection = function ( hasLockfile: detection.hasLockfile, targetCompute: context.targetCompute, setupTrigger: context.trigger, + setupMode: context.setupMode, // Omit pythonVersion entirely when unknown -- recordEvent serializes an // explicit `undefined` to the string "undefined", which would pollute // the schema for users without a resolved interpreter. diff --git a/packages/databricks-vscode/src/telemetry/pythonSetupExtensions.test.ts b/packages/databricks-vscode/src/telemetry/pythonSetupExtensions.test.ts index 105d4b7f7..fc09de0e5 100644 --- a/packages/databricks-vscode/src/telemetry/pythonSetupExtensions.test.ts +++ b/packages/databricks-vscode/src/telemetry/pythonSetupExtensions.test.ts @@ -554,4 +554,35 @@ describe(__filename, () => { ]); }); }); + + describe("recordManualSetupOptout", () => { + it("emits the opt-out event with scope and source", () => { + const {telemetry, events} = makeTelemetry(); + + telemetry.recordManualSetupOptout({ + scope: "workspace", + source: "error_popup", + }); + + expect(events).to.have.length(1); + expect(events[0].name).to.equal("python_env.manual_setup.optout"); + expect(events[0].props).to.deep.equal({ + "version": "1.0", + "event.scope": "workspace", + "event.source": "error_popup", + }); + }); + + it("carries the global / command_palette variant through", () => { + const {telemetry, events} = makeTelemetry(); + + telemetry.recordManualSetupOptout({ + scope: "global", + source: "command_palette", + }); + + expect(events[0].props["event.scope"]).to.equal("global"); + expect(events[0].props["event.source"]).to.equal("command_palette"); + }); + }); }); diff --git a/packages/databricks-vscode/src/telemetry/pythonSetupExtensions.ts b/packages/databricks-vscode/src/telemetry/pythonSetupExtensions.ts index 2d1238500..79a71e5d6 100644 --- a/packages/databricks-vscode/src/telemetry/pythonSetupExtensions.ts +++ b/packages/databricks-vscode/src/telemetry/pythonSetupExtensions.ts @@ -7,6 +7,8 @@ import { PythonSetupFailurePhase, PythonSetupFlow, PythonSetupMode, + PythonSetupOptoutScope, + PythonSetupOptoutSource, PythonSetupOutcome, PythonSetupRunTrigger, TargetCompute, @@ -242,9 +244,21 @@ declare module "." { * itself the adoption-rate denominator. */ recordPythonSetupAdoption(report: PythonSetupAdoption): void; + /** + * Record that the user opted out of automated uv-native setup (switched + * `databricks.python.environmentSetup` to `manual`). Call only after the + * setting write succeeds, so the count reflects actual opt-outs. + */ + recordManualSetupOptout(report: ManualSetupOptout): void; } } +/** A manual-setup opt-out, reduced to the categorical fields we report. */ +export interface ManualSetupOptout { + scope: PythonSetupOptoutScope; + source: PythonSetupOptoutSource; +} + // Both payloads below name every field explicitly instead of spreading the // caller's object. Spreading a *variable* switches off TypeScript's // excess-property check, so any field later added to PythonSetupAttempt / @@ -360,3 +374,14 @@ Telemetry.prototype.recordPythonSetupAdoption = function ( currentTargetType: report.currentTargetType, }); }; + +Telemetry.prototype.recordManualSetupOptout = function ( + report: ManualSetupOptout +): void { + // Named explicitly (not spread) for the same allowlist reason as the + // emitters above; both fields are required categorical strings. + this.recordEvent(Events.PYTHON_ENV_MANUAL_SETUP_OPTOUT, { + scope: report.scope, + source: report.source, + }); +}; From 6968a87e0bd49792880d863e244c6e08a9036c69 Mon Sep 17 00:00:00 2001 From: "@rugpanov" Date: Thu, 3 Sep 2026 15:34:34 +0200 Subject: [PATCH 2/3] refactor(python-setup): address opt-out telemetry review *Why* Multi-source review of the opt-out telemetry raised: the event over-counted when the setting was already manual; the command-handler glue was untested; and two type names collided / broke the repo's OptOut casing. *What* - Record the opt-out only on a genuine auto->manual transition (not on every successful write), so repeat invocations while already manual don't inflate the count. Extract the handler into optOutOfAutomatedPythonSetup with injected deps, covered by unit tests (transition, already-manual, no-folder, write failure). - Rename the reported-flow type PythonEnvSetupMode -> ReportedSetupMode to avoid collision with PythonEnvironmentSetupMode / PythonSetupMode. - Rename Optout -> OptOut in identifiers to match the repo's optOutOfInstallPrompt. - Clarify the setupMode field comment: emitted per (trigger, project); measure prevalence by distinct user, not raw event share. - Use import type for the now type-only telemetry imports. *Verification* - yarn build (tsc) and yarn test:lint clean; yarn test:unit green (1065 passing). Co-authored-by: Isaac --- packages/databricks-vscode/src/extension.ts | 51 +++------ .../utils/manualSetupOptOut.test.ts | 105 ++++++++++++++++++ .../python-setup/utils/manualSetupOptOut.ts | 75 +++++++++++++ .../src/python-setup/utils/pythonSetupGate.ts | 18 +-- .../src/telemetry/constants.ts | 22 ++-- .../src/telemetry/packageManagerExtensions.ts | 4 +- .../telemetry/pythonSetupExtensions.test.ts | 6 +- .../src/telemetry/pythonSetupExtensions.ts | 22 ++-- 8 files changed, 235 insertions(+), 68 deletions(-) create mode 100644 packages/databricks-vscode/src/python-setup/utils/manualSetupOptOut.test.ts create mode 100644 packages/databricks-vscode/src/python-setup/utils/manualSetupOptOut.ts diff --git a/packages/databricks-vscode/src/extension.ts b/packages/databricks-vscode/src/extension.ts index c51ba90b0..6d859297f 100644 --- a/packages/databricks-vscode/src/extension.ts +++ b/packages/databricks-vscode/src/extension.ts @@ -1,6 +1,5 @@ import { commands, - ConfigurationTarget, debug, env, ExtensionContext, @@ -60,6 +59,7 @@ import {PythonSetupDriftManager} from "./python-setup/controllers/PythonSetupDri import {PythonSetupAdoptionManager} from "./python-setup/controllers/PythonSetupAdoptionManager"; import {SetupCompute} from "./python-setup/controllers/PythonSetupEnvironmentSetup"; import {venvInterpreterPath} from "./python-setup/utils/venvInterpreterPath"; +import {optOutOfAutomatedPythonSetup} from "./python-setup/utils/manualSetupOptOut"; import { INSTALL_UV_COMMAND_ID, USE_MANUAL_SETUP_COMMAND_ID, @@ -77,7 +77,7 @@ import {DatabricksEnvFileManager} from "./file-managers/DatabricksEnvFileManager import {getContextMetadata, Telemetry, toUserMetadata} from "./telemetry"; import "./telemetry/commandExtensions"; import "./telemetry/pythonSetupExtensions"; -import {Events, Metadata, PythonSetupOptoutSource} from "./telemetry/constants"; +import {Events, Metadata, PythonSetupOptOutSource} from "./telemetry/constants"; import {EnvironmentDependenciesInstaller} from "./language/EnvironmentDependenciesInstaller"; import {setDbnbCellLimits} from "./language/notebooks/DatabricksNbCellLimits"; import {DbConnectStatusBarButton} from "./language/DbConnectStatusBarButton"; @@ -1076,44 +1076,27 @@ export async function activate( // folder is open we fall back to Global so the write still lands. telemetry.registerCommand( USE_MANUAL_SETUP_COMMAND_ID, - async (arg?: {source?: PythonSetupOptoutSource}) => { + async (arg?: {source?: PythonSetupOptOutSource}) => { // The E_FETCH popup passes {source: "error_popup"}; a palette // invocation passes nothing, so anything else reads as palette. - const source: PythonSetupOptoutSource = + const source: PythonSetupOptOutSource = arg?.source === "error_popup" ? "error_popup" : "command_palette"; - const hasFolder = (workspace.workspaceFolders?.length ?? 0) > 0; - const target = hasFolder - ? ConfigurationTarget.Workspace - : ConfigurationTarget.Global; - try { - await workspaceConfigs.setPythonEnvironmentSetup( - "manual", - target - ); - } catch (e) { - // Don't claim success if the write failed — the user would - // otherwise believe automated setup is off when it is not. - await window.showErrorMessage( - `Could not update databricks.python.environmentSetup: ${ - e instanceof Error ? e.message : String(e) - }` - ); - return; - } - // Record only after the write lands, so the count reflects real - // opt-outs. scope mirrors the ConfigurationTarget written. - telemetry.recordManualSetupOptout({ - scope: hasFolder ? "workspace" : "global", - source, + await optOutOfAutomatedPythonSetup(source, { + currentMode: () => workspaceConfigs.pythonEnvironmentSetup, + hasFolder: () => + (workspace.workspaceFolders?.length ?? 0) > 0, + setManual: (target) => + workspaceConfigs.setPythonEnvironmentSetup( + "manual", + target + ), + recordOptOut: (report) => + telemetry.recordManualSetupOptOut(report), + showError: (m) => window.showErrorMessage(m), + showInfo: (m) => window.showInformationMessage(m), }); - // Match the message to the scope actually written: "this project" - // for Workspace, "globally" for the no-folder Global fallback. - const scope = hasFolder ? "for this project" : "globally"; - await window.showInformationMessage( - `Automated Python environment setup is now off ${scope} ("databricks.python.environmentSetup": "manual"). Your existing interpreter will be used as-is.` - ); } ), // E_UV_MISSING's "Install uv" button: run uv's official installer in a diff --git a/packages/databricks-vscode/src/python-setup/utils/manualSetupOptOut.test.ts b/packages/databricks-vscode/src/python-setup/utils/manualSetupOptOut.test.ts new file mode 100644 index 000000000..886da56e1 --- /dev/null +++ b/packages/databricks-vscode/src/python-setup/utils/manualSetupOptOut.test.ts @@ -0,0 +1,105 @@ +import {expect} from "chai"; +import {ConfigurationTarget} from "vscode"; +import {PythonEnvironmentSetupMode} from "../../vscode-objs/WorkspaceConfigs"; +import { + PythonSetupOptOutScope, + PythonSetupOptOutSource, +} from "../../telemetry/constants"; +import { + ManualSetupOptOutDeps, + optOutOfAutomatedPythonSetup, +} from "./manualSetupOptOut"; + +/** + * Build the injected deps with recording spies. `currentMode` and `hasFolder` + * are configurable; `setManual` fails when `writeError` is set. + */ +function makeDeps(opts: { + currentMode?: PythonEnvironmentSetupMode; + hasFolder?: boolean; + writeError?: Error; +}) { + const recorded: { + scope: PythonSetupOptOutScope; + source: PythonSetupOptOutSource; + }[] = []; + const written: ConfigurationTarget[] = []; + const errors: string[] = []; + const infos: string[] = []; + const deps: ManualSetupOptOutDeps = { + currentMode: () => opts.currentMode ?? "auto", + hasFolder: () => opts.hasFolder ?? true, + setManual: async (target) => { + if (opts.writeError) { + throw opts.writeError; + } + written.push(target); + }, + recordOptOut: (report) => recorded.push(report), + showError: async (m) => errors.push(m), + showInfo: async (m) => infos.push(m), + }; + return {deps, recorded, written, errors, infos}; +} + +describe("optOutOfAutomatedPythonSetup", () => { + it("records the opt-out on a real auto->manual transition (workspace)", async () => { + const {deps, recorded, written, infos} = makeDeps({ + currentMode: "auto", + hasFolder: true, + }); + + await optOutOfAutomatedPythonSetup("error_popup", deps); + + expect(written).to.deep.equal([ConfigurationTarget.Workspace]); + expect(recorded).to.deep.equal([ + {scope: "workspace", source: "error_popup"}, + ]); + expect(infos[0]).to.contain("for this project"); + }); + + it("writes and confirms Global scope when no folder is open", async () => { + const {deps, recorded, written, infos} = makeDeps({ + currentMode: "auto", + hasFolder: false, + }); + + await optOutOfAutomatedPythonSetup("command_palette", deps); + + expect(written).to.deep.equal([ConfigurationTarget.Global]); + expect(recorded).to.deep.equal([ + {scope: "global", source: "command_palette"}, + ]); + expect(infos[0]).to.contain("globally"); + }); + + it("does NOT record when already manual (no transition), but still confirms", async () => { + const {deps, recorded, written, infos} = makeDeps({ + currentMode: "manual", + hasFolder: true, + }); + + await optOutOfAutomatedPythonSetup("command_palette", deps); + + // Idempotent write still happens, but no opt-out is counted. + expect(written).to.deep.equal([ConfigurationTarget.Workspace]); + expect(recorded).to.have.length(0); + expect(infos).to.have.length(1); + }); + + it("does NOT record and surfaces the error when the write fails", async () => { + const {deps, recorded, errors, infos} = makeDeps({ + currentMode: "auto", + writeError: new Error("EACCES"), + }); + + await optOutOfAutomatedPythonSetup("error_popup", deps); + + expect(recorded).to.have.length(0); + expect(infos).to.have.length(0); + expect(errors[0]).to.contain( + "Could not update databricks.python.environmentSetup" + ); + expect(errors[0]).to.contain("EACCES"); + }); +}); diff --git a/packages/databricks-vscode/src/python-setup/utils/manualSetupOptOut.ts b/packages/databricks-vscode/src/python-setup/utils/manualSetupOptOut.ts new file mode 100644 index 000000000..c2a72bd3f --- /dev/null +++ b/packages/databricks-vscode/src/python-setup/utils/manualSetupOptOut.ts @@ -0,0 +1,75 @@ +import {ConfigurationTarget} from "vscode"; +import type {PythonEnvironmentSetupMode} from "../../vscode-objs/WorkspaceConfigs"; +import type { + PythonSetupOptOutScope, + PythonSetupOptOutSource, +} from "../../telemetry/constants"; + +/** + * Collaborators for {@link optOutOfAutomatedPythonSetup}, injected so the + * decision logic (transition detection, scope, record-on-success) is unit-tested + * without VS Code globals. + */ +export interface ManualSetupOptOutDeps { + /** The effective setup mode, read before the write to detect a transition. */ + currentMode: () => PythonEnvironmentSetupMode; + /** Whether a workspace folder is open; decides the scope and target. */ + hasFolder: () => boolean; + /** Persist `"manual"` at the given configuration target. */ + setManual: (target: ConfigurationTarget) => PromiseLike; + /** Record the opt-out (called only on a genuine auto->manual transition). */ + recordOptOut: (report: { + scope: PythonSetupOptOutScope; + source: PythonSetupOptOutSource; + }) => void; + /** Surface a write failure to the user. */ + showError: (message: string) => PromiseLike; + /** Confirm the new state to the user. */ + showInfo: (message: string) => PromiseLike; +} + +/** + * Turn automated Python-environment setup off (set + * `databricks.python.environmentSetup` to `manual`) so an existing interpreter + * is used as-is. Writes to Workspace scope when a folder is open, else Global so + * the write still lands. + * + * The opt-out is recorded only on a real `auto -> manual` transition: re-running + * the command while already manual re-writes idempotently and reassures the user, + * but does not inflate the opt-out count. Nothing is recorded when the write + * fails — the user is told, and the setting is unchanged. + */ +export async function optOutOfAutomatedPythonSetup( + source: PythonSetupOptOutSource, + deps: ManualSetupOptOutDeps +): Promise { + const wasManual = deps.currentMode() === "manual"; + const hasFolder = deps.hasFolder(); + const target = hasFolder + ? ConfigurationTarget.Workspace + : ConfigurationTarget.Global; + try { + await deps.setManual(target); + } catch (e) { + // Don't claim success if the write failed — the user would otherwise + // believe automated setup is off when it is not. + await deps.showError( + `Could not update databricks.python.environmentSetup: ${ + e instanceof Error ? e.message : String(e) + }` + ); + return; + } + if (!wasManual) { + deps.recordOptOut({ + scope: hasFolder ? "workspace" : "global", + source, + }); + } + // Match the message to the scope actually written: "this project" for + // Workspace, "globally" for the no-folder Global fallback. + const scope = hasFolder ? "for this project" : "globally"; + await deps.showInfo( + `Automated Python environment setup is now off ${scope} ("databricks.python.environmentSetup": "manual"). Your existing interpreter will be used as-is.` + ); +} diff --git a/packages/databricks-vscode/src/python-setup/utils/pythonSetupGate.ts b/packages/databricks-vscode/src/python-setup/utils/pythonSetupGate.ts index fdddf7c26..3192ccf98 100644 --- a/packages/databricks-vscode/src/python-setup/utils/pythonSetupGate.ts +++ b/packages/databricks-vscode/src/python-setup/utils/pythonSetupGate.ts @@ -89,14 +89,16 @@ export function isUvSetupSuitable(detection: SuitabilityDetection): boolean { } /** - * The environment-setup flow actually in effect for a session, as reported on - * `python_env.setup.detected`. Distinct from the `auto | manual` - * `databricks.python.environmentSetup` setting ({@link PythonEnvironmentSetupMode}): - * `manual` maps to `fallback-pip` (the user opted out; an existing interpreter is - * used as-is), while `auto` splits by uv-suitability into `uv` (uv-native flow) - * or `pip` (legacy/non-uv flow — the exact manager stays in `primaryManager`). + * The environment-setup flow actually in effect, as reported on + * `python_env.setup.detected`. Named distinctly from the `auto | manual` + * `databricks.python.environmentSetup` setting ({@link PythonEnvironmentSetupMode}) + * and the `default | constraints-only` provisioning `PythonSetupMode`, since all + * three sit on the same telemetry surface: `manual` maps to `fallback-pip` (the + * user opted out; an existing interpreter is used as-is), while `auto` splits by + * uv-suitability into `uv` (uv-native flow) or `pip` (legacy/non-uv flow — the + * exact manager stays in `primaryManager`). */ -export type PythonEnvSetupMode = "uv" | "pip" | "fallback-pip"; +export type ReportedSetupMode = "uv" | "pip" | "fallback-pip"; /** * Resolve the reported setup mode from the user's setting and the detected @@ -107,7 +109,7 @@ export type PythonEnvSetupMode = "uv" | "pip" | "fallback-pip"; export function resolveSetupMode( setting: PythonEnvironmentSetupMode, detection: SuitabilityDetection -): PythonEnvSetupMode { +): ReportedSetupMode { if (setting === "manual") { return "fallback-pip"; } diff --git a/packages/databricks-vscode/src/telemetry/constants.ts b/packages/databricks-vscode/src/telemetry/constants.ts index b02ce6313..1e9b2a322 100644 --- a/packages/databricks-vscode/src/telemetry/constants.ts +++ b/packages/databricks-vscode/src/telemetry/constants.ts @@ -107,19 +107,19 @@ export type {PackageManager, PrimaryManager, InterpreterSource}; // The effective setup-flow union lives with the uv-suitability gate that derives // it (the single source of truth), re-exported here so the event schema and the // resolver can never drift. Type-only, so no runtime dependency/cycle. -import type {PythonEnvSetupMode} from "../python-setup/utils/pythonSetupGate"; -export type {PythonEnvSetupMode}; +import type {ReportedSetupMode} from "../python-setup/utils/pythonSetupGate"; +export type {ReportedSetupMode}; /** * Which configuration scope the manual opt-out was written to: `workspace` when * a project folder is open, `global` for the no-folder fallback. */ -export type PythonSetupOptoutScope = "workspace" | "global"; +export type PythonSetupOptOutScope = "workspace" | "global"; /** * Where the manual opt-out was triggered: the `error_popup` "Use manual setup" * button on the E_FETCH failure, or the `command_palette`. */ -export type PythonSetupOptoutSource = "error_popup" | "command_palette"; +export type PythonSetupOptOutSource = "error_popup" | "command_palette"; /** The compute targeted at the time of detection. */ export type TargetCompute = ComputeType | "none"; @@ -472,7 +472,7 @@ export class EventTypes { hasLockfile: boolean; targetCompute: TargetCompute; setupTrigger: SetupTrigger; - setupMode: PythonEnvSetupMode; + setupMode: ReportedSetupMode; }> = { comment: "The Python package/environment manager(s) detected for a project at setup time. " + @@ -509,15 +509,17 @@ export class EventTypes { }, setupMode: { comment: - "The environment-setup flow in effect this session: uv (uv-native setup), pip " + - "(legacy/non-uv flow; the exact manager is in primaryManager), or fallback-pip " + + "The environment-setup flow in effect at this detection: uv (uv-native setup), " + + "pip (legacy/non-uv flow; the exact manager is in primaryManager), or fallback-pip " + "(user opted out via databricks.python.environmentSetup=manual). The auto|manual " + - "setting is recoverable: fallback-pip <=> manual, uv|pip <=> auto", + "setting is recoverable: fallback-pip <=> manual, uv|pip <=> auto. Emitted per " + + "(trigger, project) like the rest of this event, so measure prevalence by distinct " + + "user (common.vscodemachineid), not raw event share", }, }; [Events.PYTHON_ENV_MANUAL_SETUP_OPTOUT]: EventType<{ - scope: PythonSetupOptoutScope; - source: PythonSetupOptoutSource; + scope: PythonSetupOptOutScope; + source: PythonSetupOptOutSource; }> = { comment: "The user opted out of automated uv-native Python setup by switching " + diff --git a/packages/databricks-vscode/src/telemetry/packageManagerExtensions.ts b/packages/databricks-vscode/src/telemetry/packageManagerExtensions.ts index 22f91d515..e967331d6 100644 --- a/packages/databricks-vscode/src/telemetry/packageManagerExtensions.ts +++ b/packages/databricks-vscode/src/telemetry/packageManagerExtensions.ts @@ -1,5 +1,5 @@ import {Events, Telemetry} from "."; -import {TargetCompute, SetupTrigger, PythonEnvSetupMode} from "./constants"; +import type {TargetCompute, SetupTrigger, ReportedSetupMode} from "./constants"; import {PackageManagerDetection} from "../language/packageManagerDetection"; /** @@ -11,7 +11,7 @@ export interface PackageManagerDetectionContext { pythonVersion?: string; targetCompute: TargetCompute; trigger: SetupTrigger; - setupMode: PythonEnvSetupMode; + setupMode: ReportedSetupMode; } declare module "." { diff --git a/packages/databricks-vscode/src/telemetry/pythonSetupExtensions.test.ts b/packages/databricks-vscode/src/telemetry/pythonSetupExtensions.test.ts index fc09de0e5..24696dfda 100644 --- a/packages/databricks-vscode/src/telemetry/pythonSetupExtensions.test.ts +++ b/packages/databricks-vscode/src/telemetry/pythonSetupExtensions.test.ts @@ -555,11 +555,11 @@ describe(__filename, () => { }); }); - describe("recordManualSetupOptout", () => { + describe("recordManualSetupOptOut", () => { it("emits the opt-out event with scope and source", () => { const {telemetry, events} = makeTelemetry(); - telemetry.recordManualSetupOptout({ + telemetry.recordManualSetupOptOut({ scope: "workspace", source: "error_popup", }); @@ -576,7 +576,7 @@ describe(__filename, () => { it("carries the global / command_palette variant through", () => { const {telemetry, events} = makeTelemetry(); - telemetry.recordManualSetupOptout({ + telemetry.recordManualSetupOptOut({ scope: "global", source: "command_palette", }); diff --git a/packages/databricks-vscode/src/telemetry/pythonSetupExtensions.ts b/packages/databricks-vscode/src/telemetry/pythonSetupExtensions.ts index 79a71e5d6..68c03066f 100644 --- a/packages/databricks-vscode/src/telemetry/pythonSetupExtensions.ts +++ b/packages/databricks-vscode/src/telemetry/pythonSetupExtensions.ts @@ -1,5 +1,5 @@ import {Events, Telemetry} from "."; -import { +import type { ComputeType, PrimaryManager, PythonSetupDriftTrigger, @@ -7,8 +7,8 @@ import { PythonSetupFailurePhase, PythonSetupFlow, PythonSetupMode, - PythonSetupOptoutScope, - PythonSetupOptoutSource, + PythonSetupOptOutScope, + PythonSetupOptOutSource, PythonSetupOutcome, PythonSetupRunTrigger, TargetCompute, @@ -246,17 +246,17 @@ declare module "." { recordPythonSetupAdoption(report: PythonSetupAdoption): void; /** * Record that the user opted out of automated uv-native setup (switched - * `databricks.python.environmentSetup` to `manual`). Call only after the - * setting write succeeds, so the count reflects actual opt-outs. + * `databricks.python.environmentSetup` to `manual`). Call only on a + * genuine auto->manual transition, so the count reflects real opt-outs. */ - recordManualSetupOptout(report: ManualSetupOptout): void; + recordManualSetupOptOut(report: ManualSetupOptOut): void; } } /** A manual-setup opt-out, reduced to the categorical fields we report. */ -export interface ManualSetupOptout { - scope: PythonSetupOptoutScope; - source: PythonSetupOptoutSource; +export interface ManualSetupOptOut { + scope: PythonSetupOptOutScope; + source: PythonSetupOptOutSource; } // Both payloads below name every field explicitly instead of spreading the @@ -375,8 +375,8 @@ Telemetry.prototype.recordPythonSetupAdoption = function ( }); }; -Telemetry.prototype.recordManualSetupOptout = function ( - report: ManualSetupOptout +Telemetry.prototype.recordManualSetupOptOut = function ( + report: ManualSetupOptOut ): void { // Named explicitly (not spread) for the same allowlist reason as the // emitters above; both fields are required categorical strings. From 9cf28f767b26a79b7a54ac6e9e255b6a5208c60b Mon Sep 17 00:00:00 2001 From: "@rugpanov" Date: Thu, 3 Sep 2026 15:41:59 +0200 Subject: [PATCH 3/3] docs(python-setup): correct opt-out helper docstring The helper injects out the VS Code UI globals (window/workspace/commands) but still imports the ConfigurationTarget enum as a runtime value, so its tests run under the extension-host harness. Reword the doc-comment that claimed 'without VS Code globals'. Co-authored-by: Isaac --- .../src/python-setup/utils/manualSetupOptOut.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/databricks-vscode/src/python-setup/utils/manualSetupOptOut.ts b/packages/databricks-vscode/src/python-setup/utils/manualSetupOptOut.ts index c2a72bd3f..f5ef4335c 100644 --- a/packages/databricks-vscode/src/python-setup/utils/manualSetupOptOut.ts +++ b/packages/databricks-vscode/src/python-setup/utils/manualSetupOptOut.ts @@ -8,7 +8,9 @@ import type { /** * Collaborators for {@link optOutOfAutomatedPythonSetup}, injected so the * decision logic (transition detection, scope, record-on-success) is unit-tested - * without VS Code globals. + * without the VS Code UI globals (window/workspace/commands). The tests still run + * under the extension-host harness because the module imports the + * `ConfigurationTarget` enum as a runtime value. */ export interface ManualSetupOptOutDeps { /** The effective setup mode, read before the write to detect a transition. */