Skip to content

Commit 0c581b2

Browse files
authored
Add telemetry for imports (microsoft#4736)
For #4718 <!-- If an item below does not apply to you, then go ahead and check it off as "done" and strikethrough the text, e.g.: - [x] ~Has unit tests & system/integration tests~ --> - [x] Pull request represents a single change (i.e. not fixing disparate/unrelated things in a single PR) - [x] Title summarizes what is changing - [x] Has a [news entry](https://github.com/Microsoft/vscode-python/tree/master/news) file (remember to thank yourself!) - [ ] Has sufficient logging. - [ ] Has telemetry for enhancements. - [x] Unit tests & system/integration tests are added/updated - [ ] [Test plan](https://github.com/Microsoft/vscode-python/blob/master/.github/test_plan.md) is updated as appropriate - [ ] [`package-lock.json`](https://github.com/Microsoft/vscode-python/blob/master/package-lock.json) has been regenerated by running `npm install` (if dependencies have changed) - [ ] The wiki is updated with any design decisions/details.
1 parent 90264dc commit 0c581b2

16 files changed

Lines changed: 478 additions & 21 deletions

File tree

news/1 Enhancements/4718.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Add telemetry around imports

src/client/common/serviceRegistry.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
// Licensed under the MIT License.
33
import { IHttpClient } from '../activation/types';
44
import { IServiceManager } from '../ioc/types';
5+
import { ImportTracker } from '../telemetry/importTracker';
6+
import { IImportTracker } from '../telemetry/types';
57
import { ApplicationEnvironment } from './application/applicationEnvironment';
68
import { ApplicationShell } from './application/applicationShell';
79
import { CommandManager } from './application/commandManager';
@@ -112,4 +114,5 @@ export function registerTypes(serviceManager: IServiceManager) {
112114

113115
serviceManager.addSingleton<IAsyncDisposableRegistry>(IAsyncDisposableRegistry, AsyncDisposableRegistry);
114116
serviceManager.addSingleton<IMultiStepInputFactory>(IMultiStepInputFactory, MultiStepInputFactory);
117+
serviceManager.addSingleton<IImportTracker>(IImportTracker, ImportTracker);
115118
}

src/client/datascience/history.ts

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ export class History implements IHistory {
7373
private changeHandler: IDisposable | undefined;
7474
private messageListener : HistoryMessageListener;
7575
private id : string;
76+
private executeEvent: EventEmitter<string> = new EventEmitter<string>();
7677

7778
constructor(
7879
@inject(ILiveShareApi) private liveShare : ILiveShareApi,
@@ -142,6 +143,10 @@ export class History implements IHistory {
142143
return this.closedEvent.event;
143144
}
144145

146+
public get onExecutedCode() : Event<string> {
147+
return this.executeEvent.event;
148+
}
149+
145150
public addCode(code: string, file: string, line: number, editor?: TextEditor) : Promise<void> {
146151
// Call the internal method.
147152
return this.submitCode(code, file, line, undefined, editor);
@@ -490,6 +495,9 @@ export class History implements IHistory {
490495
// Attempt to evaluate this cell in the jupyter notebook
491496
const observable = this.jupyterServer.executeObservable(code, file, line, id, false);
492497

498+
// Indicate we executed some code
499+
this.executeEvent.fire(code);
500+
493501
// Sign up for cell changes
494502
observable.subscribe(
495503
(cells: ICell[]) => {

src/client/datascience/historyProvider.ts

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
'use strict';
44
import { inject, injectable } from 'inversify';
55
import * as uuid from 'uuid/v4';
6+
import { Disposable, Event, EventEmitter } from 'vscode';
67
import * as vsls from 'vsls/vscode';
78

89
import { ILiveShareApi, IWorkspaceService } from '../common/application/types';
@@ -25,6 +26,8 @@ export class HistoryProvider implements IHistoryProvider, IAsyncDisposable {
2526
private postOffice : PostOffice;
2627
private id: string;
2728
private pendingSyncs : Map<string, ISyncData> = new Map<string, ISyncData>();
29+
private executedCode: EventEmitter<string> = new EventEmitter<string>();
30+
private activeHistoryExecuteHandler: Disposable | undefined;
2831
constructor(
2932
@inject(ILiveShareApi) liveShare: ILiveShareApi,
3033
@inject(IServiceContainer) private serviceContainer: IServiceContainer,
@@ -55,6 +58,10 @@ export class HistoryProvider implements IHistoryProvider, IAsyncDisposable {
5558
return this.activeHistory;
5659
}
5760

61+
public get onExecutedCode() : Event<string> {
62+
return this.executedCode.event;
63+
}
64+
5865
public async getOrCreateActive() : Promise<IHistory> {
5966
if (!this.activeHistory) {
6067
this.activeHistory = await this.create();
@@ -104,6 +111,8 @@ export class HistoryProvider implements IHistoryProvider, IAsyncDisposable {
104111
const handler = result.closed(this.onHistoryClosed);
105112
this.disposables.push(result);
106113
this.disposables.push(handler);
114+
this.activeHistoryExecuteHandler = result.onExecutedCode(this.onHistoryExecute);
115+
this.disposables.push(this.activeHistoryExecuteHandler);
107116
await result.ready;
108117
return result;
109118
}
@@ -151,6 +160,10 @@ export class HistoryProvider implements IHistoryProvider, IAsyncDisposable {
151160
private onHistoryClosed = (history: IHistory) => {
152161
if (this.activeHistory === history) {
153162
this.activeHistory = undefined;
163+
if (this.activeHistoryExecuteHandler) {
164+
this.activeHistoryExecuteHandler.dispose();
165+
this.activeHistoryExecuteHandler = undefined;
166+
}
154167
}
155168
}
156169

@@ -171,4 +184,8 @@ export class HistoryProvider implements IHistoryProvider, IAsyncDisposable {
171184
return Promise.resolve();
172185
}
173186

187+
private onHistoryExecute = (code: string) => {
188+
this.executedCode.fire(code);
189+
}
190+
174191
}

src/client/datascience/types.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,7 @@ export interface INotebookExporter extends Disposable {
122122

123123
export const IHistoryProvider = Symbol('IHistoryProvider');
124124
export interface IHistoryProvider {
125+
onExecutedCode: Event<string>;
125126
getActive() : IHistory | undefined;
126127
getOrCreateActive(): Promise<IHistory>;
127128
getNotebookOptions() : Promise<INotebookServerOptions>;
@@ -131,6 +132,7 @@ export const IHistory = Symbol('IHistory');
131132
export interface IHistory extends Disposable {
132133
closed: Event<IHistory>;
133134
ready: Promise<void>;
135+
onExecutedCode: Event<string>;
134136
show() : Promise<void>;
135137
addCode(code: string, file: string, line: number, editor?: TextEditor) : Promise<void>;
136138
// tslint:disable-next-line:no-any

src/client/extension.ts

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ import { ISortImportsEditingProvider } from './providers/types';
9797
import { activateUpdateSparkLibraryProvider } from './providers/updateSparkLibraryProvider';
9898
import { sendTelemetryEvent } from './telemetry';
9999
import { EventName } from './telemetry/constants';
100-
import { EditorLoadTelemetry } from './telemetry/types';
100+
import { EditorLoadTelemetry, IImportTracker } from './telemetry/types';
101101
import { registerTypes as commonRegisterTerminalTypes } from './terminals/serviceRegistry';
102102
import { ICodeExecutionManager, ITerminalAutoActivation } from './terminals/types';
103103
import { TEST_OUTPUT_CHANNEL } from './unittests/common/constants';
@@ -162,6 +162,10 @@ async function activateUnsafe(context: ExtensionContext): Promise<IExtensionApi>
162162
const dataScience = serviceManager.get<IDataScience>(IDataScience);
163163
dataScience.activate().ignoreErrors();
164164

165+
// Activate import tracking
166+
const importTracker = serviceManager.get<IImportTracker>(IImportTracker);
167+
importTracker.activate().ignoreErrors();
168+
165169
context.subscriptions.push(new LinterCommands(serviceManager));
166170
const linterProvider = new LinterProvider(context, serviceManager);
167171
context.subscriptions.push(linterProvider);

src/client/telemetry/constants.ts

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,10 +66,44 @@ export enum EventName {
6666

6767
SELECT_LINTER = 'LINTING.SELECT',
6868

69-
LINTER_NOT_INSTALLED_PROMPT = 'LINTER_NOT_INSTALLED_PROMPT'
69+
LINTER_NOT_INSTALLED_PROMPT = 'LINTER_NOT_INSTALLED_PROMPT',
70+
KNOWN_IMPORT_FROM_FILE = 'KNOWN_IMPORT_FROM_FILE',
71+
KNOWN_IMPORT_FROM_EXECUTION = 'KNOWN_IMPORT_FROM_EXECUTION'
7072
}
7173

7274
export enum PlatformErrors {
7375
FailedToParseVersion = 'FailedToParseVersion',
7476
FailedToDetermineOS = 'FailedToDetermineOS'
7577
}
78+
79+
export enum KnownImports {
80+
// Don't change the order of these as they are the value returned in telemetry and changing
81+
// the order will break old telemetry data.
82+
//
83+
// This list was generated from here: https://activewizards.com/blog/top-20-python-libraries-for-data-science-in-2018/
84+
pandas,
85+
numpy,
86+
matlplotlib,
87+
scipy,
88+
sklearn,
89+
statsmodels,
90+
seaborn,
91+
plotly,
92+
bokeh,
93+
pydot,
94+
xgboost,
95+
lightgbm,
96+
catboost,
97+
eli5,
98+
tensorflow,
99+
pytorch,
100+
keras,
101+
distkeras,
102+
elephas,
103+
pyspark,
104+
nltk,
105+
spacy,
106+
gensim,
107+
scrapy,
108+
sparkdl
109+
}
Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
'use strict';
4+
import '../common/extensions';
5+
6+
import { inject, injectable } from 'inversify';
7+
import * as path from 'path';
8+
import { TextDocument } from 'vscode';
9+
10+
import { sendTelemetryEvent } from '.';
11+
import { sleep } from '../../test/core';
12+
import { IDocumentManager } from '../common/application/types';
13+
import { IHistoryProvider } from '../datascience/types';
14+
import { ICodeExecutionManager } from '../terminals/types';
15+
import { EventName, KnownImports } from './constants';
16+
import { IImportTracker } from './types';
17+
18+
const ImportRegEx = /^(?!['"#]).*from\s+([a-zA-Z0-9_\.]+)\s+import.*(?!['"])|^(?!['"#]).*import\s+([a-zA-Z0-9_\., ]+).*(?!['"])/;
19+
const MAX_DOCUMENT_LINES = 1000;
20+
21+
@injectable()
22+
export class ImportTracker implements IImportTracker {
23+
24+
private knownImportsMatch: RegExp;
25+
private sentMatches: Set<string> = new Set<string>();
26+
27+
constructor(
28+
@inject(IDocumentManager) private documentManager: IDocumentManager,
29+
@inject(IHistoryProvider) private historyProvider: IHistoryProvider,
30+
@inject(ICodeExecutionManager) private executionManager: ICodeExecutionManager
31+
) {
32+
// Construct a regex that will match known imports
33+
let matchString = '';
34+
Object.keys(KnownImports).forEach(k => {
35+
const val = KnownImports[k];
36+
if (typeof val === 'string') {
37+
matchString += `${val}|`;
38+
}
39+
});
40+
this.knownImportsMatch = new RegExp(matchString.slice(0, matchString.length - 1), 'g');
41+
42+
// Sign up for document open/save events so we can track known imports
43+
this.documentManager.onDidOpenTextDocument((t) => this.onOpenedOrSavedDocument(t));
44+
this.documentManager.onDidSaveTextDocument((t) => this.onOpenedOrSavedDocument(t));
45+
46+
// Sign up for history execution events (user can input code here too)
47+
this.historyProvider.onExecutedCode(c => this.onExecutedCode(c));
48+
49+
// Sign up for terminal execution events (user can send code to the terminal)
50+
// However we won't get any text typed directly into the terminal. Not part of the VS code API
51+
// Could potentially hook stdin? Not sure that's possible.
52+
this.executionManager.onExecutedCode(c => this.onExecutedCode(c));
53+
}
54+
55+
public async activate() : Promise<void> {
56+
// Act like all of our open documents just opened. Don't do this now though. We don't want
57+
// to hold up the activate.
58+
await sleep(1000);
59+
this.documentManager.textDocuments.forEach(d => this.onOpenedOrSavedDocument(d));
60+
}
61+
62+
private onOpenedOrSavedDocument(document: TextDocument) {
63+
// Make sure this is a python file.
64+
if (path.extname(document.fileName) === '.py')
65+
{
66+
// Parse the contents of the document, looking for import matches on each line
67+
const lines = document.getText().splitLines({ trim: true, removeEmptyEntries: true });
68+
this.lookForImports(lines.slice(0, Math.min(lines.length, MAX_DOCUMENT_LINES)), EventName.KNOWN_IMPORT_FROM_FILE);
69+
}
70+
}
71+
72+
private onExecutedCode(code: string) {
73+
const lines = code.splitLines({ trim: true, removeEmptyEntries: true });
74+
this.lookForImports(lines, EventName.KNOWN_IMPORT_FROM_EXECUTION);
75+
}
76+
77+
private lookForImports(lines: string[], eventName: string) {
78+
// Use a regex to parse each line, looking for imports
79+
const matches: Set<string> = new Set<string>();
80+
for (const s of lines) {
81+
const match = ImportRegEx.exec(s);
82+
if (match && match.length > 2) {
83+
// Could be a from or a straight import. from is the first entry.
84+
const actual = match[1] ? match[1] : match[2];
85+
86+
// See if this matches any known imports
87+
let knownMatch: RegExpExecArray = this.knownImportsMatch.exec(actual);
88+
while (knownMatch) {
89+
knownMatch.forEach(val => {
90+
// Skip if already sent this telemetry
91+
if (!this.sentMatches.has(val)) {
92+
matches.add(val);
93+
}
94+
});
95+
knownMatch = this.knownImportsMatch.exec(actual);
96+
}
97+
98+
// Reset search.
99+
this.knownImportsMatch.lastIndex = 0;
100+
}
101+
}
102+
103+
// For each unique match, emit a new telemetry event.
104+
matches.forEach(s => {
105+
sendTelemetryEvent(
106+
eventName === EventName.KNOWN_IMPORT_FROM_FILE ? EventName.KNOWN_IMPORT_FROM_FILE : EventName.KNOWN_IMPORT_FROM_EXECUTION,
107+
0,
108+
{ import: s });
109+
this.sentMatches.add(s);
110+
});
111+
}
112+
}

src/client/telemetry/index.ts

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
// Copyright (c) Microsoft Corporation. All rights reserved.
22
// Licensed under the MIT License.
3-
43
// tslint:disable:no-reference no-any import-name no-any function-name
54
/// <reference path="./vscode-extension-telemetry.d.ts" />
65
import { basename as pathBasename, sep as pathSep } from 'path';
76
import * as stackTrace from 'stack-trace';
87
import TelemetryReporter from 'vscode-extension-telemetry';
8+
99
import { EXTENSION_ROOT_DIR, isTestExecution, PVSC_EXTENSION_ID } from '../common/constants';
1010
import { StopWatch } from '../common/utils/stopWatch';
1111
import { Telemetry } from '../datascience/constants';
@@ -53,7 +53,7 @@ function isTelemetrySupported(): boolean {
5353
return false;
5454
}
5555
}
56-
let telemetryReporter: TelemetryReporter;
56+
let telemetryReporter: TelemetryReporter | undefined;
5757
function getTelemetryReporter() {
5858
if (!isTestExecution() && telemetryReporter) {
5959
return telemetryReporter;
@@ -73,6 +73,10 @@ function getTelemetryReporter() {
7373
return (telemetryReporter = new reporter(extensionId, extensionVersion, aiKey));
7474
}
7575

76+
export function clearTelemetryReporter() {
77+
telemetryReporter = undefined;
78+
}
79+
7680
export function sendTelemetryEvent<P extends IEventNamePropertyMapping, E extends keyof P>(
7781
eventName: E,
7882
durationMs?: Record<string, number> | number,
@@ -106,6 +110,13 @@ export function sendTelemetryEvent<P extends IEventNamePropertyMapping, E extend
106110
reporter.sendTelemetryEvent('ERROR', customProperties, measures);
107111
}
108112
reporter.sendTelemetryEvent((eventName as any) as string, customProperties, measures);
113+
114+
// Enable this to debug telemetry. To be discussed whether or not we want this all of the time.
115+
// try {
116+
// traceInfo(`Telemetry: ${eventName} : ${JSON.stringify(customProperties)}`);
117+
// } catch {
118+
// noop();
119+
// }
109120
}
110121

111122
// tslint:disable-next-line:no-any function-name
@@ -243,7 +254,7 @@ function getCallsite(frame: stackTrace.StackFrame) {
243254
}
244255

245256
// Map all events to their properties
246-
interface IEventNamePropertyMapping {
257+
export interface IEventNamePropertyMapping {
247258
[EventName.COMPLETION]: never | undefined;
248259
[EventName.COMPLETION_ADD_BRACKETS]: { enabled: boolean };
249260
[EventName.DEBUGGER]: DebuggerTelemetry;
@@ -262,6 +273,8 @@ interface IEventNamePropertyMapping {
262273
[EventName.FORMAT_SORT_IMPORTS]: never | undefined;
263274
[EventName.GO_TO_OBJECT_DEFINITION]: never | undefined;
264275
[EventName.HOVER_DEFINITION]: never | undefined;
276+
[EventName.KNOWN_IMPORT_FROM_FILE] : { import: string };
277+
[EventName.KNOWN_IMPORT_FROM_EXECUTION] : { import: string };
265278
[EventName.LINTER_NOT_INSTALLED_PROMPT]: LinterInstallPromptTelemetry;
266279
[EventName.LINTING]: LintingTelemetry;
267280
[EventName.PLATFORM_INFO]: Platform;

src/client/telemetry/types.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -190,3 +190,8 @@ export type InterpreterActivation = {
190190
pythonVersion?: string;
191191
interpreterType: InterpreterType;
192192
};
193+
194+
export const IImportTracker = Symbol('IImportTracker');
195+
export interface IImportTracker {
196+
activate() : Promise<void>;
197+
}

0 commit comments

Comments
 (0)