Skip to content

Commit a2d6ac7

Browse files
kimadelinekimadeline
authored
Log experiments run via ExP in the output panel (#12742)
* Log experiments run via ExP in the output panel * Fix tests * It worked on my machine ;_; * Filter out non-PVSC experiments * Fix tests * Clarification. * Clarification 2. * Case-insentitive filtering
1 parent 9404460 commit a2d6ac7

3 files changed

Lines changed: 83 additions & 15 deletions

File tree

‎news/3 Code Health/12656.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Log ExP experiments the user belongs to in the output panel.

‎src/client/common/experiments/service.ts‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,20 @@ import { getExperimentationService, IExperimentationService, TargetPopulation }
99
import { sendTelemetryEvent } from '../../telemetry';
1010
import { EventName } from '../../telemetry/constants';
1111
import { IApplicationEnvironment } from '../application/types';
12-
import { PVSC_EXTENSION_ID } from '../constants';
13-
import { GLOBAL_MEMENTO, IConfigurationService, IExperimentService, IMemento, IPythonSettings } from '../types';
12+
import { PVSC_EXTENSION_ID, STANDARD_OUTPUT_CHANNEL } from '../constants';
13+
import {
14+
GLOBAL_MEMENTO,
15+
IConfigurationService,
16+
IExperimentService,
17+
IMemento,
18+
IOutputChannel,
19+
IPythonSettings
20+
} from '../types';
21+
import { Experiments } from '../utils/localize';
1422
import { ExperimentationTelemetry } from './telemetry';
1523

24+
const EXP_MEMENTO_KEY = 'VSCode.ABExp.FeatureData';
25+
1626
@injectable()
1727
export class ExperimentService implements IExperimentService {
1828
/**
@@ -30,7 +40,8 @@ export class ExperimentService implements IExperimentService {
3040
constructor(
3141
@inject(IConfigurationService) readonly configurationService: IConfigurationService,
3242
@inject(IApplicationEnvironment) private readonly appEnvironment: IApplicationEnvironment,
33-
@inject(IMemento) @named(GLOBAL_MEMENTO) readonly globalState: Memento
43+
@inject(IMemento) @named(GLOBAL_MEMENTO) private readonly globalState: Memento,
44+
@inject(IOutputChannel) @named(STANDARD_OUTPUT_CHANNEL) private readonly output: IOutputChannel
3445
) {
3546
this.settings = configurationService.getSettings(undefined);
3647

@@ -61,8 +72,10 @@ export class ExperimentService implements IExperimentService {
6172
this.appEnvironment.packageJson.version!,
6273
targetPopulation,
6374
telemetryReporter,
64-
globalState
75+
this.globalState
6576
);
77+
78+
this.logExperiments();
6679
}
6780

6881
public async inExperiment(experiment: string): Promise<boolean> {
@@ -90,4 +103,15 @@ export class ExperimentService implements IExperimentService {
90103

91104
return this.experimentationService.isCachedFlightEnabled(experiment);
92105
}
106+
107+
private logExperiments() {
108+
const experiments = this.globalState.get<{ features: string[] }>(EXP_MEMENTO_KEY, { features: [] });
109+
110+
experiments.features.forEach((exp) => {
111+
// Filter out experiments groups that are not from the Python extension.
112+
if (exp.toLowerCase().startsWith('python')) {
113+
this.output.appendLine(Experiments.inGroup().format(exp));
114+
}
115+
});
116+
}
93117
}

‎src/test/common/experiments/service.unit.test.ts‎

Lines changed: 54 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,18 @@
55

66
import { assert } from 'chai';
77
import * as sinon from 'sinon';
8-
import { instance, mock, when } from 'ts-mockito';
8+
import { anything, instance, mock, when } from 'ts-mockito';
99
import * as tasClient from 'vscode-tas-client';
1010
import { ApplicationEnvironment } from '../../../client/common/application/applicationEnvironment';
1111
import { Channel, IApplicationEnvironment } from '../../../client/common/application/types';
1212
import { ConfigurationService } from '../../../client/common/configuration/service';
1313
import { ExperimentService } from '../../../client/common/experiments/service';
1414
import { IConfigurationService } from '../../../client/common/types';
15+
import { Experiments } from '../../../client/common/utils/localize';
1516
import * as Telemetry from '../../../client/telemetry';
1617
import { EventName } from '../../../client/telemetry/constants';
1718
import { PVSC_EXTENSION_ID_FOR_TESTS } from '../../constants';
19+
import { MockOutputChannel } from '../../mockClasses';
1820
import { MockMemento } from '../../mocks/mementos';
1921

2022
suite('Experimentation service', () => {
@@ -23,15 +25,18 @@ suite('Experimentation service', () => {
2325
let configurationService: IConfigurationService;
2426
let appEnvironment: IApplicationEnvironment;
2527
let globalMemento: MockMemento;
28+
let outputChannel: MockOutputChannel;
2629

2730
setup(() => {
2831
configurationService = mock(ConfigurationService);
2932
appEnvironment = mock(ApplicationEnvironment);
3033
globalMemento = new MockMemento();
34+
outputChannel = new MockOutputChannel('');
3135
});
3236

3337
teardown(() => {
3438
sinon.restore();
39+
Telemetry._resetSharedProperties();
3540
});
3641

3742
function configureSettings(enabled: boolean, optInto: string[], optOutFrom: string[]) {
@@ -59,7 +64,12 @@ suite('Experimentation service', () => {
5964
configureApplicationEnvironment('stable', extensionVersion);
6065

6166
// tslint:disable-next-line: no-unused-expression
62-
new ExperimentService(instance(configurationService), instance(appEnvironment), globalMemento);
67+
new ExperimentService(
68+
instance(configurationService),
69+
instance(appEnvironment),
70+
globalMemento,
71+
outputChannel
72+
);
6373

6474
sinon.assert.calledWithExactly(
6575
getExperimentationServiceStub,
@@ -78,7 +88,12 @@ suite('Experimentation service', () => {
7888
configureApplicationEnvironment('insiders', extensionVersion);
7989

8090
// tslint:disable-next-line: no-unused-expression
81-
new ExperimentService(instance(configurationService), instance(appEnvironment), globalMemento);
91+
new ExperimentService(
92+
instance(configurationService),
93+
instance(appEnvironment),
94+
globalMemento,
95+
outputChannel
96+
);
8297

8398
sinon.assert.calledWithExactly(
8499
getExperimentationServiceStub,
@@ -99,7 +114,8 @@ suite('Experimentation service', () => {
99114
const experimentService = new ExperimentService(
100115
instance(configurationService),
101116
instance(appEnvironment),
102-
globalMemento
117+
globalMemento,
118+
outputChannel
103119
);
104120

105121
assert.deepEqual(experimentService._optInto, ['Foo - experiment']);
@@ -113,11 +129,32 @@ suite('Experimentation service', () => {
113129
const experimentService = new ExperimentService(
114130
instance(configurationService),
115131
instance(appEnvironment),
116-
globalMemento
132+
globalMemento,
133+
outputChannel
117134
);
118135

119136
assert.deepEqual(experimentService._optOutFrom, ['Foo - experiment']);
120137
});
138+
139+
test('Experiment data in Memento storage should be logged if it starts with "python"', () => {
140+
const experiments = ['ExperimentOne', 'pythonExperiment'];
141+
globalMemento = mock(MockMemento);
142+
configureSettings(true, [], []);
143+
configureApplicationEnvironment('stable', extensionVersion);
144+
// tslint:disable-next-line: no-any
145+
when(globalMemento.get(anything(), anything())).thenReturn({ features: experiments } as any);
146+
147+
// tslint:disable-next-line: no-unused-expression
148+
new ExperimentService(
149+
instance(configurationService),
150+
instance(appEnvironment),
151+
instance(globalMemento),
152+
outputChannel
153+
);
154+
const output = `${Experiments.inGroup().format('pythonExperiment')}\n`;
155+
156+
assert.equal(outputChannel.output, output);
157+
});
121158
});
122159

123160
suite('In-experiment check', () => {
@@ -153,7 +190,8 @@ suite('Experimentation service', () => {
153190
const experimentService = new ExperimentService(
154191
instance(configurationService),
155192
instance(appEnvironment),
156-
globalMemento
193+
globalMemento,
194+
outputChannel
157195
);
158196
const result = await experimentService.inExperiment(experiment);
159197

@@ -168,7 +206,8 @@ suite('Experimentation service', () => {
168206
const experimentService = new ExperimentService(
169207
instance(configurationService),
170208
instance(appEnvironment),
171-
globalMemento
209+
globalMemento,
210+
outputChannel
172211
);
173212
const result = await experimentService.inExperiment(experiment);
174213

@@ -183,7 +222,8 @@ suite('Experimentation service', () => {
183222
const experimentService = new ExperimentService(
184223
instance(configurationService),
185224
instance(appEnvironment),
186-
globalMemento
225+
globalMemento,
226+
outputChannel
187227
);
188228
const result = await experimentService.inExperiment(experiment);
189229

@@ -202,7 +242,8 @@ suite('Experimentation service', () => {
202242
const experimentService = new ExperimentService(
203243
instance(configurationService),
204244
instance(appEnvironment),
205-
globalMemento
245+
globalMemento,
246+
outputChannel
206247
);
207248
const result = await experimentService.inExperiment(experiment);
208249

@@ -221,7 +262,8 @@ suite('Experimentation service', () => {
221262
const experimentService = new ExperimentService(
222263
instance(configurationService),
223264
instance(appEnvironment),
224-
globalMemento
265+
globalMemento,
266+
outputChannel
225267
);
226268
const result = await experimentService.inExperiment(experiment);
227269

@@ -240,7 +282,8 @@ suite('Experimentation service', () => {
240282
const experimentService = new ExperimentService(
241283
instance(configurationService),
242284
instance(appEnvironment),
243-
globalMemento
285+
globalMemento,
286+
outputChannel
244287
);
245288
const result = await experimentService.inExperiment(experiment);
246289

0 commit comments

Comments
 (0)