Add Environments integration decision telemetry

src/client/envExt/api.internal.ts

@@ -16,6 +16,7 @@ import { executeCommand } from '../common/vscodeApis/commandApis';
 import { getConfiguration, getWorkspaceFolders } from '../common/vscodeApis/workspaceApis';
 import { traceError, traceLog } from '../logging';
 import { Interpreters } from '../common/utils/localize';
+import type { EnvsIntegrationDecisionReason, EnvsIntegrationDecisionTelemetry } from '../telemetry/types';
 
 export const ENVS_EXTENSION_ID = 'ms-python.vscode-python-envs';
 
@@ -54,18 +55,55 @@ export function shouldEnvExtHandleActivation(): boolean {
     return true;
 }
 
+function getEnvExtensionDecisionInputs(): Pick<
+    EnvsIntegrationDecisionTelemetry,
+    'envsAvailableToHostNow' | 'envsActiveNow' | 'envsResolvedSettingNow'
+> {
+    const config = getConfiguration('python');
+    const extension = getExtension(ENVS_EXTENSION_ID);
+    return {
+        envsAvailableToHostNow: !!extension,
+        envsActiveNow: extension?.isActive ?? false,
+        envsResolvedSettingNow: config?.get<boolean>('useEnvironmentsExtension', false) ?? false,
+    };
+}
+
 let _useExt: boolean | undefined;
+let _envsDecisionReason: EnvsIntegrationDecisionReason | undefined;
 export function useEnvExtension(): boolean {
     if (_useExt !== undefined) {
         return _useExt;
     }
-    const config = getConfiguration('python');
-    const inExpSetting = config?.get<boolean>('useEnvironmentsExtension', false) ?? false;
+    const { envsAvailableToHostNow, envsResolvedSettingNow } = getEnvExtensionDecisionInputs();
     // If extension is installed and in experiment, then use it.
-    _useExt = !!getExtension(ENVS_EXTENSION_ID) && inExpSetting;
+    _useExt = envsAvailableToHostNow && envsResolvedSettingNow;
+    _envsDecisionReason = !envsAvailableToHostNow
+        ? 'extensionUnavailable'
+        : envsResolvedSettingNow
+        ? 'enabled'
+        : 'resolvedSettingFalse';
     return _useExt;
 }
 
+/**
+ * Reports the cached integration decision inputs and their current values without initializing or changing the cache.
+ */
+export function getEnvExtensionDecisionTelemetry(): EnvsIntegrationDecisionTelemetry {
+    return {
+        envsDecisionReason: _envsDecisionReason,
+        ...getEnvExtensionDecisionInputs(),
+        envsCachedDecision: _useExt,
+    };
+}
+
+/**
+ * Resets the cached integration decision for unit tests.
+ */
+export function _resetEnvExtensionDecisionCache(): void {
+    _useExt = undefined;
+    _envsDecisionReason = undefined;
+}
+
 const onDidChangeEnvironmentEnvExtEmitter: EventEmitter<DidChangeEnvironmentEventArgs> = new EventEmitter<
     DidChangeEnvironmentEventArgs
 >();

src/client/envExt/telemetry.ts

@@ -3,6 +3,7 @@
 
 import { IDisposableRegistry } from '../common/types';
 import { getConfiguration, getWorkspaceFolders, onDidChangeConfiguration } from '../common/vscodeApis/workspaceApis';
+import { getEnvExtensionDecisionTelemetry } from './api.internal';
 import { sendTelemetryEvent } from '../telemetry';
 import { EventName } from '../telemetry/constants';
 import { EnvsExplicitFalseScope } from '../telemetry/types';
@@ -45,6 +46,7 @@ export function registerEnvironmentsExtensionTelemetry(disposables: IDisposableR
             if (event.affectsConfiguration('python.useEnvironmentsExtension')) {
                 sendTelemetryEvent(EventName.ENVIRONMENTS_EXTENSION_SETTING_CHANGED, undefined, {
                     envsExplicitFalseScope: getEnvsExplicitFalseScope(),
+                    ...getEnvExtensionDecisionTelemetry(),
                 });
             }
         }),

src/client/startupTelemetry.ts

@@ -16,9 +16,19 @@ import { sendTelemetryEvent } from './telemetry';
 import { EventName } from './telemetry/constants';
 import { EditorLoadTelemetry } from './telemetry/types';
 import { IStartupDurations } from './types';
-import { useEnvExtension } from './envExt/api.internal';
+import { getEnvExtensionDecisionTelemetry, useEnvExtension } from './envExt/api.internal';
 import { getEnvsExplicitFalseScope } from './envExt/telemetry';
 
+function getEnvExtensionStartupTelemetry() {
+    const telemetry = getEnvExtensionDecisionTelemetry();
+    return {
+        envsDecisionReason: telemetry.envsDecisionReason,
+        envsAvailableToHostNow: telemetry.envsAvailableToHostNow,
+        envsActiveNow: telemetry.envsActiveNow,
+        envsResolvedSettingNow: telemetry.envsResolvedSettingNow,
+    };
+}
+
 export async function sendStartupTelemetry(
     activatedPromise: Promise<any>,
     durations: IStartupDurations,
@@ -99,6 +109,7 @@ async function getActivationTelemetryProps(
             terminal: terminalShellType,
             isFirstSession,
             envsExplicitFalseScope: getEnvsExplicitFalseScope(),
+            ...getEnvExtensionStartupTelemetry(),
         };
     }
     const interpreterService = serviceContainer.get<IInterpreterService>(IInterpreterService);
@@ -157,5 +168,6 @@ async function getActivationTelemetryProps(
         isFirstSession,
         usingEnvironmentsExtension,
         envsExplicitFalseScope: getEnvsExplicitFalseScope(),
+        ...getEnvExtensionStartupTelemetry(),
     };
 }

src/client/telemetry/index.ts

@@ -13,7 +13,7 @@ import { EnvironmentType, PythonEnvironment } from '../pythonEnvironments/info';
 import { TensorBoardPromptSelection } from '../tensorBoard/constants';
 import { EventName } from './constants';
 import type { UnitTestRunFailureCategory } from './constants';
-import type { EnvsExplicitFalseScope, TestTool } from './types';
+import type { EnvsExplicitFalseScope, EnvsIntegrationDecisionReason, TestTool } from './types';
 
 /**
  * Checks whether telemetry is supported.
@@ -360,7 +360,11 @@ export interface IEventNamePropertyMapping {
           "usingglobalinterpreter" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "luabud" },
           "isfirstsession" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "luabud" },
           "usingenvironmentsextension" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eduardovil" },
-          "envsexplicitfalsescope" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" }
+          "envsexplicitfalsescope" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+          "envsdecisionreason" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+          "envsavailabletohostnow" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+          "envsactivenow" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+          "envsresolvedsettingnow" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" }
        }
      */
     [EventName.EDITOR_LOAD]: {
@@ -416,18 +420,44 @@ export interface IEventNamePropertyMapping {
          * 'none' excludes defaults; 'multiple' means more than one of user, workspace, or folder.
          */
         envsExplicitFalseScope?: EnvsExplicitFalseScope;
+        /**
+         * The inputs that produced the cached Python Environments integration decision.
+         */
+        envsDecisionReason?: EnvsIntegrationDecisionReason;
+        /**
+         * Whether Python Environments is currently available to this extension host.
+         */
+        envsAvailableToHostNow?: boolean;
+        /**
+         * Whether Python Environments is currently active in this extension host.
+         */
+        envsActiveNow?: boolean;
+        /**
+         * The current resolved value of python.useEnvironmentsExtension.
+         */
+        envsResolvedSettingNow?: boolean;
     };
     /**
      * Reports explicit-false scopes after a python.useEnvironmentsExtension configuration change.
      * This is a configuration notification, not proof of user intent or a changed integration decision.
      */
     /* __GDPR__
        "environments_extension.setting_changed" : {
-          "envsexplicitfalsescope" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" }
+          "envsexplicitfalsescope" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+          "envsdecisionreason" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+          "envsavailabletohostnow" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+          "envsactivenow" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+          "envsresolvedsettingnow" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" },
+          "envscacheddecision" : { "classification": "SystemMetaData", "purpose": "FeatureInsight", "owner": "eleanorjboyd" }
        }
      */
     [EventName.ENVIRONMENTS_EXTENSION_SETTING_CHANGED]: {
         envsExplicitFalseScope: EnvsExplicitFalseScope;
+        envsDecisionReason?: EnvsIntegrationDecisionReason;
+        envsAvailableToHostNow: boolean;
+        envsActiveNow: boolean;
+        envsResolvedSettingNow: boolean;
+        envsCachedDecision?: boolean;
     };
     /**
      * Telemetry event sent when substituting Environment variables to calculate value of variables

src/client/telemetry/types.ts

@@ -10,6 +10,16 @@ export type EditorLoadTelemetry = IEventNamePropertyMapping[EventName.EDITOR_LOA
 
 export type EnvsExplicitFalseScope = 'none' | 'user' | 'workspace' | 'folder' | 'multiple';
 
+export type EnvsIntegrationDecisionReason = 'enabled' | 'extensionUnavailable' | 'resolvedSettingFalse';
+
+export type EnvsIntegrationDecisionTelemetry = {
+    envsDecisionReason?: EnvsIntegrationDecisionReason;
+    envsAvailableToHostNow: boolean;
+    envsActiveNow: boolean;
+    envsResolvedSettingNow: boolean;
+    envsCachedDecision?: boolean;
+};
+
 export type PythonInterpreterTelemetry = IEventNamePropertyMapping[EventName.PYTHON_INTERPRETER];
 export type TestTool = 'pytest' | 'unittest';
 export type TestRunTelemetry = IEventNamePropertyMapping[EventName.UNITTEST_RUN];

src/test/envExt/api.internal.unit.test.ts

@@ -0,0 +1,109 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+import { assert } from 'chai';
+import * as sinon from 'sinon';
+import * as typemoq from 'typemoq';
+import { Extension, WorkspaceConfiguration } from 'vscode';
+import * as extensionsApi from '../../client/common/vscodeApis/extensionsApi';
+import * as workspaceApis from '../../client/common/vscodeApis/workspaceApis';
+import {
+    _resetEnvExtensionDecisionCache,
+    ENVS_EXTENSION_ID,
+    getEnvExtensionDecisionTelemetry,
+    useEnvExtension,
+} from '../../client/envExt/api.internal';
+
+function configuration(resolvedSetting: boolean): WorkspaceConfiguration {
+    const config = typemoq.Mock.ofType<WorkspaceConfiguration>();
+    config.setup((c) => c.get<boolean>('useEnvironmentsExtension', false)).returns(() => resolvedSetting);
+    return config.object;
+}
+
+suite('Python Environments integration decision telemetry', () => {
+    let getConfiguration: sinon.SinonStub;
+    let getExtension: sinon.SinonStub;
+
+    setup(() => {
+        _resetEnvExtensionDecisionCache();
+        getConfiguration = sinon.stub(workspaceApis, 'getConfiguration').returns(configuration(false));
+        getExtension = sinon.stub(extensionsApi, 'getExtension').returns(undefined);
+    });
+
+    teardown(() => {
+        _resetEnvExtensionDecisionCache();
+        sinon.restore();
+    });
+
+    const cases = [
+        {
+            name: 'extension unavailable',
+            available: false,
+            active: false,
+            resolvedSetting: true,
+            decision: false,
+            reason: 'extensionUnavailable',
+        },
+        {
+            name: 'resolved setting false',
+            available: true,
+            active: true,
+            resolvedSetting: false,
+            decision: false,
+            reason: 'resolvedSettingFalse',
+        },
+        {
+            name: 'integration enabled',
+            available: true,
+            active: true,
+            resolvedSetting: true,
+            decision: true,
+            reason: 'enabled',
+        },
+    ] as const;
+
+    cases.forEach(({ name, available, active, resolvedSetting, decision, reason }) => {
+        test(`captures ${name}`, () => {
+            getConfiguration.returns(configuration(resolvedSetting));
+            getExtension.returns(available ? ({ isActive: active } as Extension<unknown>) : undefined);
+
+            assert.strictEqual(useEnvExtension(), decision);
+            assert.deepEqual(getEnvExtensionDecisionTelemetry(), {
+                envsDecisionReason: reason,
+                envsAvailableToHostNow: available,
+                envsActiveNow: active,
+                envsResolvedSettingNow: resolvedSetting,
+                envsCachedDecision: decision,
+            });
+            sinon.assert.alwaysCalledWithExactly(getExtension, ENVS_EXTENSION_ID);
+        });
+    });
+
+    test('reports live inputs without changing a stale cached decision', () => {
+        assert.isFalse(useEnvExtension());
+        getConfiguration.returns(configuration(true));
+        getExtension.returns({ isActive: true } as Extension<unknown>);
+
+        assert.deepEqual(getEnvExtensionDecisionTelemetry(), {
+            envsDecisionReason: 'extensionUnavailable',
+            envsAvailableToHostNow: true,
+            envsActiveNow: true,
+            envsResolvedSettingNow: true,
+            envsCachedDecision: false,
+        });
+        assert.isFalse(useEnvExtension());
+    });
+
+    test('does not initialize the cached decision when telemetry is requested', () => {
+        getConfiguration.returns(configuration(true));
+        getExtension.returns({ isActive: false } as Extension<unknown>);
+
+        assert.deepEqual(getEnvExtensionDecisionTelemetry(), {
+            envsDecisionReason: undefined,
+            envsAvailableToHostNow: true,
+            envsActiveNow: false,
+            envsResolvedSettingNow: true,
+            envsCachedDecision: undefined,
+        });
+    });
+});

src/test/envExt/telemetry.unit.test.ts

@@ -8,6 +8,7 @@ import * as typemoq from 'typemoq';
 import { ConfigurationChangeEvent, EventEmitter, Uri, WorkspaceConfiguration, WorkspaceFolder } from 'vscode';
 import { IDisposableRegistry } from '../../client/common/types';
 import * as workspaceApis from '../../client/common/vscodeApis/workspaceApis';
+import * as envExt from '../../client/envExt/api.internal';
 import { getEnvsExplicitFalseScope, registerEnvironmentsExtensionTelemetry } from '../../client/envExt/telemetry';
 import * as telemetry from '../../client/telemetry';
 import { EventName } from '../../client/telemetry/constants';
@@ -44,13 +45,21 @@ suite('Environments extension telemetry', () => {
     let sendTelemetryEvent: sinon.SinonStub;
     let changes: EventEmitter<ConfigurationChangeEvent>;
     let disposables: IDisposableRegistry;
+    const decisionTelemetry = {
+        envsDecisionReason: 'resolvedSettingFalse' as const,
+        envsAvailableToHostNow: true,
+        envsActiveNow: true,
+        envsResolvedSettingNow: false,
+        envsCachedDecision: false,
+    };
 
     setup(() => {
         getConfiguration = sinon.stub(workspaceApis, 'getConfiguration').returns(configuration({}));
         getWorkspaceFolders = sinon.stub(workspaceApis, 'getWorkspaceFolders').returns(undefined);
         sendTelemetryEvent = sinon.stub(telemetry, 'sendTelemetryEvent');
         changes = new EventEmitter<ConfigurationChangeEvent>();
         sinon.stub(workspaceApis, 'onDidChangeConfiguration').callsFake((listener) => changes.event(listener));
+        sinon.stub(envExt, 'getEnvExtensionDecisionTelemetry').returns(decisionTelemetry);
         disposables = [];
     });
 
@@ -170,7 +179,7 @@ suite('Environments extension telemetry', () => {
             ['user', 'workspace', 'none'].map((envsExplicitFalseScope) => [
                 EventName.ENVIRONMENTS_EXTENSION_SETTING_CHANGED,
                 undefined,
-                { envsExplicitFalseScope },
+                { envsExplicitFalseScope, ...decisionTelemetry },
             ]),
         );
     });
@@ -185,7 +194,7 @@ suite('Environments extension telemetry', () => {
             sendTelemetryEvent,
             EventName.ENVIRONMENTS_EXTENSION_SETTING_CHANGED,
             undefined,
-            { envsExplicitFalseScope: 'folder' },
+            { envsExplicitFalseScope: 'folder', ...decisionTelemetry },
         );
     });
 

src/test/startupTelemetry.unit.test.ts

@@ -73,6 +73,13 @@ suite('Startup Telemetry - hasUserDefinedPythonPath()', async () => {
                 .returns(() => interpreterPathService.object);
             sinon.stub(constants, 'isTestExecution').returns(false);
             sinon.stub(envExt, 'useEnvExtension').returns(false);
+            sinon.stub(envExt, 'getEnvExtensionDecisionTelemetry').returns({
+                envsDecisionReason: 'resolvedSettingFalse',
+                envsAvailableToHostNow: true,
+                envsActiveNow: true,
+                envsResolvedSettingNow: false,
+                envsCachedDecision: false,
+            });
             sinon.stub(envExtTelemetry, 'getEnvsExplicitFalseScope').returns('workspace');
             sendTelemetryEvent = sinon.stub(telemetry, 'sendTelemetryEvent');
             durations = {
@@ -101,11 +108,19 @@ suite('Startup Telemetry - hasUserDefinedPythonPath()', async () => {
                     sendTelemetryEvent,
                     EventName.EDITOR_LOAD,
                     durations,
-                    sinon.match({ envsExplicitFalseScope: 'workspace', isFirstSession: false }),
+                    sinon.match({
+                        envsExplicitFalseScope: 'workspace',
+                        envsDecisionReason: 'resolvedSettingFalse',
+                        envsAvailableToHostNow: true,
+                        envsActiveNow: true,
+                        envsResolvedSettingNow: false,
+                        isFirstSession: false,
+                    }),
                 );
                 if (isTrusted) {
                     expect(sendTelemetryEvent.firstCall.args[2].usingEnvironmentsExtension).to.equal(false);
                 }
+                expect(sendTelemetryEvent.firstCall.args[2]).not.to.have.property('envsCachedDecision');
             });
         });
     });