From 070021e57d13e4bbeffff556f24594f86f8cfedc Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Thu, 19 Mar 2020 12:41:14 -0700 Subject: [PATCH 001/725] current interface and type (#10667) --- .../raw-kernel/enchannelJMPConnection.ts | 34 +++++++++++++++++++ src/client/datascience/types.ts | 21 ++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 src/client/datascience/raw-kernel/enchannelJMPConnection.ts diff --git a/src/client/datascience/raw-kernel/enchannelJMPConnection.ts b/src/client/datascience/raw-kernel/enchannelJMPConnection.ts new file mode 100644 index 000000000000..14f0d238b7af --- /dev/null +++ b/src/client/datascience/raw-kernel/enchannelJMPConnection.ts @@ -0,0 +1,34 @@ +import { KernelMessage } from '@jupyterlab/services'; +//import { Channels } from '@nteract/messaging'; +//import { createMainChannel } from 'enchannel-zmq-backend'; +import { IJMPConnection, IJMPConnectionInfo } from '../types'; + +export class EnchannelJMPConnection implements IJMPConnection { + //private mainChannel: Channels | undefined; + + public async connect(_connectInfo: IJMPConnectionInfo, _sessionID: string): Promise { + // tslint:disable-next-line:no-any + //this.mainChannel = await createMainChannel(connectInfo as any, undefined, sessionID); + } + public sendMessage(_message: KernelMessage.IMessage): void { + //if (this.mainChannel) { + //// jupyterlab types and enchannel types seem to have small changes + //// with how they are defined, just use an any cast for now, but they appear to be the + //// same actual object + //// tslint:disable-next-line:no-any + //this.mainChannel.next(message as any); + //} + } + public subscribe(_handlerFunc: (message: KernelMessage.IMessage) => void) { + //if (this.mainChannel) { + //// tslint:disable-next-line:no-any + //this.mainChannel.subscribe(handlerFunc as any); + //} + } + + public dispose(): void { + //if (this.mainChannel) { + //this.mainChannel.unsubscribe(); + //} + } +} diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index 4f0db713f9ec..fb289ada56cb 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -857,3 +857,24 @@ export interface INotebookProvider { */ getNotebook(server: INotebookServer, resource: Uri, options?: nbformat.INotebookMetadata): Promise; } + +// Connection info to connect to a kernel over JMP +export interface IJMPConnectionInfo { + version: number; + iopub_port: number; + shell_port: number; + stdin_port: number; + control_port: number; + signature_scheme: string; + hb_port: number; + ip: string; + key: string; + transport: string; +} + +// A service to send and recieve messages over Jupyter messaging protocol +export interface IJMPConnection extends IDisposable { + connect(connectInfo: IJMPConnectionInfo, sessionID: string): Promise; + sendMessage(message: KernelMessage.IMessage): void; + subscribe(handlerFunc: (message: KernelMessage.IMessage) => void): void; +} From df0554c9b6bddc42c5cac1ff9827e1223511040a Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Fri, 20 Mar 2020 14:15:46 -0700 Subject: [PATCH 002/725] Testable Kernel and Future (#10679) --- package-lock.json | 6 +- package.json | 2 +- .../datascience/raw-kernel/rawFuture.ts | 155 ++++++++++ .../datascience/raw-kernel/rawKernel.ts | 268 ++++++++++++++++++ src/test/datascience/execution.unit.test.ts | 2 +- ...eractiveWindowCommandListener.unit.test.ts | 2 +- .../jupyter/jupyterSession.unit.test.ts | 14 +- .../raw-kernel/rawFuture.unit.test.ts | 70 +++++ .../raw-kernel/rawKernel.unit.test.ts | 95 +++++++ src/test/linters/linterCommands.unit.test.ts | 3 +- .../services/discoveryService.unit.test.ts | 6 +- 11 files changed, 611 insertions(+), 12 deletions(-) create mode 100644 src/client/datascience/raw-kernel/rawFuture.ts create mode 100644 src/client/datascience/raw-kernel/rawKernel.ts create mode 100644 src/test/datascience/raw-kernel/rawFuture.unit.test.ts create mode 100644 src/test/datascience/raw-kernel/rawKernel.unit.test.ts diff --git a/package-lock.json b/package-lock.json index bcd06599a795..1d8f357e0cd9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19068,9 +19068,9 @@ } }, "ts-mockito": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/ts-mockito/-/ts-mockito-2.3.1.tgz", - "integrity": "sha512-chcKw0sTApwJxTyKhzbWxI4BTUJ6RStZKUVh2/mfwYqFS09PYy5pvdXZwG35QSkqT5pkdXZlYKBX196RRvEZdQ==", + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/ts-mockito/-/ts-mockito-2.5.0.tgz", + "integrity": "sha512-b3qUeMfghRq5k5jw3xNJcnU9RKhqKnRn0k9v9QkN+YpuawrFuMIiGwzFZCpdi5MHy26o7YPnK8gag2awURl3nA==", "dev": true, "requires": { "lodash": "^4.17.5" diff --git a/package.json b/package.json index 67e539502a11..3f64489147ed 100644 --- a/package.json +++ b/package.json @@ -3132,7 +3132,7 @@ "terser-webpack-plugin": "^2.3.2", "transform-loader": "^0.2.4", "ts-loader": "^5.3.0", - "ts-mockito": "^2.3.1", + "ts-mockito": "^2.5.0", "ts-node": "^8.3.0", "tsconfig-paths-webpack-plugin": "^3.2.0", "tslint": "^5.20.1", diff --git a/src/client/datascience/raw-kernel/rawFuture.ts b/src/client/datascience/raw-kernel/rawFuture.ts new file mode 100644 index 000000000000..79b67156ddf1 --- /dev/null +++ b/src/client/datascience/raw-kernel/rawFuture.ts @@ -0,0 +1,155 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +import { Kernel, KernelMessage } from '@jupyterlab/services'; +import { createDeferred, Deferred } from '../../common/utils/async'; +import { noop } from '../../common/utils/misc'; + +/* +RawFuture represents the IFuture interface that JupyterLab services returns from functions like executeRequest. +It provides an interface for getting updates on the status of the request such as reply messages or io messages +*/ +export class RawFuture< + REQUEST extends KernelMessage.IShellControlMessage, + REPLY extends KernelMessage.IShellControlMessage +> implements Kernel.IFuture { + public isDisposed: boolean = false; + public msg: REQUEST; + + private donePromise: Deferred; + private stdIn: (msg: KernelMessage.IStdinMessage) => void | PromiseLike = noop; + private ioPub: (msg: KernelMessage.IIOPubMessage) => void | PromiseLike = noop; + private reply: (msg: REPLY) => void | PromiseLike = noop; + private replyMessage: REPLY | undefined; + private disposeOnDone: boolean; + private idleSeen: boolean; + + constructor(msg: REQUEST, disposeOnDone: boolean) { + this.msg = msg; + this.donePromise = createDeferred(); + this.disposeOnDone = disposeOnDone; + this.idleSeen = false; + } + + get done(): Promise { + return this.donePromise.promise; + } + + // Message handlers that can be hooked up to for message notifications + get onStdin(): (msg: KernelMessage.IStdinMessage) => void | PromiseLike { + return this.stdIn; + } + + set onStdin(handler: (msg: KernelMessage.IStdinMessage) => void | PromiseLike) { + this.stdIn = handler; + } + + get onIOPub(): (msg: KernelMessage.IIOPubMessage) => void | PromiseLike { + return this.ioPub; + } + + set onIOPub(cb: (msg: KernelMessage.IIOPubMessage) => void | PromiseLike) { + this.ioPub = cb; + } + get onReply(): (msg: REPLY) => void | PromiseLike { + return this.reply; + } + + set onReply(handler: (msg: REPLY) => void | PromiseLike) { + this.reply = handler; + } + + // Handle a new message passed from the kernel + public async handleMessage(message: KernelMessage.IMessage): Promise { + switch (message.channel) { + case 'stdin': + await this.handleStdIn(message as KernelMessage.IStdinMessage); + break; + case 'iopub': + await this.handleIOPub(message as KernelMessage.IIOPubMessage); + break; + case 'control': + case 'shell': + await this.handleShellControl(message as KernelMessage.IShellControlMessage); + break; + default: + break; + } + } + + public dispose(): void { + if (!this.isDisposed) { + // First clear out our handlers + this.stdIn = noop; + this.ioPub = noop; + this.reply = noop; + + // Reject our done promise + this.donePromise.reject(new Error('Disposed Future')); + this.isDisposed = true; + } + } + + // RAWKERNEL: Not Implemented + public registerMessageHook(_hook: (msg: KernelMessage.IIOPubMessage) => boolean | PromiseLike): void { + throw new Error('Not yet implemented'); + } + public removeMessageHook(_hook: (msg: KernelMessage.IIOPubMessage) => boolean | PromiseLike): void { + throw new Error('Not yet implemented'); + } + public sendInputReply(_content: KernelMessage.IInputReplyMsg['content']): void { + throw new Error('Not yet implemented'); + } + + // Private Functions + + // Functions for handling specific message types + private async handleStdIn(message: KernelMessage.IStdinMessage): Promise { + // Call our handler for stdin, might just be noop + // RAWKERNEL: same channel type string != 'stdin' cast issue + // tslint:disable-next-line:no-any + await this.stdIn(message); + } + + private async handleIOPub(message: KernelMessage.IIOPubMessage): Promise { + // RAWKERNEL: Check hooks process first? + // tslint:disable-next-line:no-any + await this.ioPub(message); + + // If we get an idle status message and a reply then we are done + if (KernelMessage.isStatusMsg(message) && message.content.execution_state === 'idle') { + this.idleSeen = true; + + if (this.replyMessage) { + this.handleDone(); + } + } + } + + private async handleShellControl(message: KernelMessage.IShellControlMessage): Promise { + if (message.channel === this.msg.channel && message.parent_header) { + const parentHeader = message.parent_header as KernelMessage.IHeader; + if (parentHeader.msg_id === this.msg.header.msg_id) { + await this.handleReply(message as REPLY); + } + } + } + + private async handleReply(message: REPLY): Promise { + await this.reply(message); + + this.replyMessage = message; + + // If we've gotten an idle status message we are done now + if (this.idleSeen) { + this.handleDone(); + } + } + + private handleDone(): void { + this.donePromise.resolve(this.replyMessage); + + if (this.disposeOnDone) { + this.dispose(); + } + } +} diff --git a/src/client/datascience/raw-kernel/rawKernel.ts b/src/client/datascience/raw-kernel/rawKernel.ts new file mode 100644 index 000000000000..5e204b637f72 --- /dev/null +++ b/src/client/datascience/raw-kernel/rawKernel.ts @@ -0,0 +1,268 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +import { Kernel, KernelMessage, ServerConnection } from '@jupyterlab/services'; +import { JSONObject } from '@phosphor/coreutils'; +import { ISignal } from '@phosphor/signaling'; +import * as uuid from 'uuid/v4'; +import { IJMPConnection, IJMPConnectionInfo } from '../types'; +import { RawFuture } from './rawFuture'; + +/* +RawKernel class represents the mapping from the JupyterLab services IKernel interface +to a raw IPython kernel running on the local machine. RawKernel is in charge of taking +input request, translating them, sending them to an IPython kernel over ZMQ, then passing back the messages +*/ +export class RawKernel implements Kernel.IKernel { + // IKernel properties + get terminated(): ISignal { + throw new Error('Not yet implemented'); + } + get statusChanged(): ISignal { + throw new Error('Not yet implemented'); + } + get iopubMessage(): ISignal { + throw new Error('Not yet implemented'); + } + get unhandledMessage(): ISignal { + throw new Error('Not yet implemented'); + } + get anyMessage(): ISignal { + throw new Error('Not yet implemented'); + } + get serverSettings(): ServerConnection.ISettings { + throw new Error('Not yet implemented'); + } + + // IKernelConnection properties + get id(): string { + throw new Error('Not yet implemented'); + } + get name(): string { + throw new Error('Not yet implemented'); + } + get model(): Kernel.IModel { + throw new Error('Not yet implemented'); + } + get username(): string { + throw new Error('Not yet implemented'); + } + get clientId(): string { + throw new Error('Not yet implemented'); + } + get status(): Kernel.Status { + throw new Error('Not yet implemented'); + } + get info(): KernelMessage.IInfoReply | null { + throw new Error('Not yet implemented'); + } + get isReady(): boolean { + throw new Error('Not yet implemented'); + } + get ready(): Promise { + throw new Error('Not yet implemented'); + } + get handleComms(): boolean { + throw new Error('Not yet implemented'); + } + + public isDisposed: boolean = false; + private jmpConnection: IJMPConnection; + private sessionId: string | undefined; + + // Keep track of all of our active futures + private futures = new Map< + string, + RawFuture + >(); + + // JMP connection should be injected, but no need to yet until it actually exists + constructor(connection: IJMPConnection) { + this.jmpConnection = connection; + } + + public async connect(connectInfo: IJMPConnectionInfo) { + this.sessionId = uuid(); + await this.jmpConnection.connect(connectInfo, this.sessionId); + this.jmpConnection.subscribe(msg => { + this.handleMessage(msg).ignoreErrors(); + }); + } + + public requestExecute( + content: KernelMessage.IExecuteRequestMsg['content'], + disposeOnDone?: boolean, + _metadata?: JSONObject + ): Kernel.IShellFuture { + if (this.jmpConnection && this.sessionId) { + // Build our execution message + // Silent is supposed to be options, but in my testing the message was not passing + // correctly without it, so specifying it here with default false + const executeOptions: KernelMessage.IOptions = { + session: this.sessionId, + channel: 'shell', + msgType: 'execute_request', + username: 'vscode', + content: { ...content, silent: content.silent || false } + }; + const executeMessage = KernelMessage.createMessage(executeOptions); + + // Send off our message to our jmp connection + this.jmpConnection.sendMessage(executeMessage); + + // Create a future to watch for reply messages + const newFuture = new RawFuture( + executeMessage, + disposeOnDone || true + ); + this.futures.set( + newFuture.msg.header.msg_id, + newFuture as RawFuture + ); + + // Set our future to remove itself when disposed + const oldDispose = newFuture.dispose.bind(newFuture); + newFuture.dispose = () => { + this.futures.delete(newFuture.msg.header.msg_id); + return oldDispose(); + }; + + return newFuture; + } + + // RAWKERNEL: What should we do here? Throw? + // Probably should not get here if session is not available + throw new Error('No session available?'); + } + + // On dispose close down our connection and get rid of saved futures + public dispose(): void { + if (!this.isDisposed) { + if (this.jmpConnection) { + this.jmpConnection.dispose(); + } + + // Dispose of all our outstanding futures + this.futures.forEach(future => { + future.dispose(); + }); + this.futures.clear(); + + this.isDisposed = true; + } + } + public shutdown(): Promise { + throw new Error('Not yet implemented'); + } + public getSpec(): Promise { + throw new Error('Not yet implemented'); + } + public sendShellMessage( + _msg: KernelMessage.IShellMessage, + _expectReply?: boolean, + _disposeOnDone?: boolean + ): Kernel.IShellFuture> { + throw new Error('Not yet implemented'); + } + public sendControlMessage( + _msg: KernelMessage.IControlMessage, + _expectReply?: boolean, + _disposeOnDone?: boolean + ): Kernel.IControlFuture> { + throw new Error('Not yet implemented'); + } + public reconnect(): Promise { + throw new Error('Not yet implemented'); + } + public interrupt(): Promise { + throw new Error('Not yet implemented'); + } + public restart(): Promise { + throw new Error('Not yet implemented'); + } + public requestKernelInfo(): Promise { + throw new Error('Not yet implemented'); + } + public requestComplete( + _content: KernelMessage.ICompleteRequestMsg['content'] + ): Promise { + throw new Error('Not yet implemented'); + } + public requestInspect( + _content: KernelMessage.IInspectRequestMsg['content'] + ): Promise { + throw new Error('Not yet implemented'); + } + public requestHistory( + _content: KernelMessage.IHistoryRequestMsg['content'] + ): Promise { + throw new Error('Not yet implemented'); + } + public requestDebug( + _content: KernelMessage.IDebugRequestMsg['content'], + _disposeOnDone?: boolean + ): Kernel.IControlFuture { + throw new Error('Not yet implemented'); + } + public requestIsComplete( + _content: KernelMessage.IIsCompleteRequestMsg['content'] + ): Promise { + throw new Error('Not yet implemented'); + } + public requestCommInfo( + _content: KernelMessage.ICommInfoRequestMsg['content'] + ): Promise { + throw new Error('Not yet implemented'); + } + public sendInputReply(_content: KernelMessage.IInputReplyMsg['content']): void { + throw new Error('Not yet implemented'); + } + public connectToComm(_targetName: string, _commId?: string): Kernel.IComm { + throw new Error('Not yet implemented'); + } + public registerCommTarget( + _targetName: string, + _callback: (comm: Kernel.IComm, _msg: KernelMessage.ICommOpenMsg) => void | PromiseLike + ): void { + throw new Error('Not yet implemented'); + } + public removeCommTarget( + _targetName: string, + _callback: (comm: Kernel.IComm, _msg: KernelMessage.ICommOpenMsg) => void | PromiseLike + ): void { + throw new Error('Not yet implemented'); + } + public registerMessageHook( + _msgId: string, + _hook: (_msg: KernelMessage.IIOPubMessage) => boolean | PromiseLike + ): void { + throw new Error('Not yet implemented'); + } + public removeMessageHook( + _msgId: string, + _hook: (_msg: KernelMessage.IIOPubMessage) => boolean | PromiseLike + ): void { + throw new Error('Not yet implemented'); + } + + // Handle a new message arriving from JMP connection + private async handleMessage(message: KernelMessage.IMessage): Promise { + // RAWKERNEL: display_data messages can route based on their id here first + + // Look up in our future list and see if a future needs to be updated on this message + if (message.parent_header) { + const parentHeader = message.parent_header as KernelMessage.IHeader; + const parentFuture = this.futures.get(parentHeader.msg_id); + + if (parentFuture) { + // Let the parent future message handle it here + await parentFuture.handleMessage(message); + } else { + if (message.header.session === this.sessionId && message.channel !== 'iopub') { + // RAWKERNEL: emit unhandled + } + } + } + + // RAWKERNEL: Handle general IOpub messages + } +} diff --git a/src/test/datascience/execution.unit.test.ts b/src/test/datascience/execution.unit.test.ts index e9f8fd4f2977..861d8b2efdff 100644 --- a/src/test/datascience/execution.unit.test.ts +++ b/src/test/datascience/execution.unit.test.ts @@ -769,7 +769,7 @@ suite('Jupyter Execution', async () => { when(interpreterService.getInterpreterDetails(match('/foo/baz/python.exe'))).thenResolve(missingKernelPython); when(interpreterService.getInterpreterDetails(match('/bar/baz/python.exe'))).thenResolve(missingNotebookPython); when(interpreterService.getInterpreterDetails(argThat(o => !o.includes || !o.includes('python')))).thenReject( - 'Unknown interpreter' + ('Unknown interpreter' as any) as Error ); if (runInDocker) { when(fileSystem.readFile('/proc/self/cgroup')).thenResolve('hello docker world'); diff --git a/src/test/datascience/interactiveWindowCommandListener.unit.test.ts b/src/test/datascience/interactiveWindowCommandListener.unit.test.ts index 45b0bafc04d1..da3ed170ce08 100644 --- a/src/test/datascience/interactiveWindowCommandListener.unit.test.ts +++ b/src/test/datascience/interactiveWindowCommandListener.unit.test.ts @@ -107,7 +107,7 @@ suite('Interactive window command listener', async () => { // Setup defaults when(interpreterService.onDidChangeInterpreter).thenReturn(dummyEvent.event); when(interpreterService.getInterpreterDetails(argThat(o => !o.includes || !o.includes('python')))).thenReject( - 'Unknown interpreter' + ('Unknown interpreter' as any) as Error ); // Service container needs logger, file system, and config service diff --git a/src/test/datascience/jupyter/jupyterSession.unit.test.ts b/src/test/datascience/jupyter/jupyterSession.unit.test.ts index 0789f5ad70c0..1c52edb028ca 100644 --- a/src/test/datascience/jupyter/jupyterSession.unit.test.ts +++ b/src/test/datascience/jupyter/jupyterSession.unit.test.ts @@ -1,8 +1,14 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -import { ContentsManager, Kernel, ServerConnection, Session, SessionManager } from '@jupyterlab/services'; +import { + ContentsManager, + Kernel, + KernelMessage, + ServerConnection, + Session, + SessionManager +} from '@jupyterlab/services'; import { DefaultKernel } from '@jupyterlab/services/lib/kernel/default'; -import { KernelFutureHandler } from '@jupyterlab/services/lib/kernel/future'; import { DefaultSession } from '@jupyterlab/services/lib/session/default'; import { ISignal, Signal } from '@phosphor/commands/node_modules/@phosphor/signaling'; import { assert } from 'chai'; @@ -321,7 +327,9 @@ suite('Data Science - JupyterSession', () => { setup(executeUserCode); async function executeUserCode() { - const future = mock(KernelFutureHandler); + const future = mock< + Kernel.IFuture + >(); // tslint:disable-next-line: no-any when(future.done).thenReturn(Promise.resolve(undefined as any)); // tslint:disable-next-line: no-any diff --git a/src/test/datascience/raw-kernel/rawFuture.unit.test.ts b/src/test/datascience/raw-kernel/rawFuture.unit.test.ts new file mode 100644 index 000000000000..bc1c2532d199 --- /dev/null +++ b/src/test/datascience/raw-kernel/rawFuture.unit.test.ts @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +import { KernelMessage } from '@jupyterlab/services'; +import { expect } from 'chai'; +import * as uuid from 'uuid/v4'; +import { RawFuture } from '../../../client/datascience/raw-kernel/rawFuture'; + +// tslint:disable: max-func-body-length +suite('Data Science - RawFuture', () => { + let rawFuture: RawFuture; + let executeMessage: KernelMessage.IExecuteRequestMsg; + let sessionID: string; + + setup(() => { + sessionID = uuid(); + // Create an execute request message + const executeOptions: KernelMessage.IOptions = { + session: sessionID, + channel: 'shell', + msgType: 'execute_request', + username: 'vscode', + content: { code: 'print("hello world")' } + }; + executeMessage = KernelMessage.createMessage(executeOptions); + rawFuture = new RawFuture(executeMessage, true); + }); + + test('Check our reply message channel', async () => { + const replyOptions: KernelMessage.IOptions = { + channel: 'shell', + session: sessionID, + msgType: 'execute_reply', + content: { status: 'ok', execution_count: 1, payload: [], user_expressions: {} } + }; + const replyMessage = KernelMessage.createMessage(replyOptions); + replyMessage.parent_header = executeMessage.header; + + // Verify that the reply message matches the one we sent + rawFuture.onReply = msg => { + expect(msg.header.msg_id).to.equal(replyMessage.header.msg_id); + }; + + await rawFuture.handleMessage(replyMessage); + + // Now take the same message and mangle the parent header, + // This message should not be sent as it doesn't match the request + replyMessage.header.msg_id = uuid(); + replyMessage.parent_header.msg_id = 'junk'; + + await rawFuture.handleMessage(replyMessage); + }); + + test('Check our IOPub message channel', async () => { + const ioPubMessageOptions: KernelMessage.IOptions = { + session: sessionID, + msgType: 'stream', + channel: 'iopub', + content: { name: 'stdout', text: 'hello' } + }; + const ioPubMessage = KernelMessage.createMessage(ioPubMessageOptions); + ioPubMessage.parent_header = executeMessage.header; + + // Verify that the iopub message matches the one we sent + rawFuture.onIOPub = msg => { + expect(msg.header.msg_id).to.equal(ioPubMessage.header.msg_id); + }; + + await rawFuture.handleMessage(ioPubMessage); + }); +}); diff --git a/src/test/datascience/raw-kernel/rawKernel.unit.test.ts b/src/test/datascience/raw-kernel/rawKernel.unit.test.ts new file mode 100644 index 000000000000..2489eebf1ecc --- /dev/null +++ b/src/test/datascience/raw-kernel/rawKernel.unit.test.ts @@ -0,0 +1,95 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +import { KernelMessage } from '@jupyterlab/services'; +import { assert, expect } from 'chai'; +import { anything, deepEqual, instance, mock, verify, when } from 'ts-mockito'; +import { RawKernel } from '../../../client/datascience/raw-kernel/rawKernel'; +import { IJMPConnection, IJMPConnectionInfo } from '../../../client/datascience/types'; + +// tslint:disable: max-func-body-length +suite('Data Science - RawKernel', () => { + let rawKernel: RawKernel; + let jmpConnection: IJMPConnection; + let connectInfo: IJMPConnectionInfo; + + setup(() => { + jmpConnection = mock(); + when(jmpConnection.connect(anything(), anything())).thenResolve(); + when(jmpConnection.subscribe(anything())).thenReturn(); + rawKernel = new RawKernel(instance(jmpConnection)); + + connectInfo = { + version: 0, + transport: 'tcp', + ip: '127.0.0.1', + shell_port: 55196, + iopub_port: 55197, + stdin_port: 55198, + hb_port: 55200, + control_port: 55199, + signature_scheme: 'hmac-sha256', + key: 'adaf9032-487d222a85026db284c3d5e7' + }; + }); + + test('RawKernel connect should connect and subscribe to JMP', async () => { + await rawKernel.connect(connectInfo); + verify(jmpConnection.connect(deepEqual(connectInfo), anything())).once(); + verify(jmpConnection.subscribe(anything())).once(); + }); + + test('RawKernel dispose should dispose the jmp', async () => { + when(jmpConnection.dispose()).thenReturn(); + + await rawKernel.connect(connectInfo); + + // Dispose our kernel + rawKernel.dispose(); + + verify(jmpConnection.dispose()).once(); + assert.isTrue(rawKernel.isDisposed); + }); + + test('RawKernel requestExecute should pass a valid execute message to JMP', async () => { + when(jmpConnection.sendMessage(anything())).thenReturn(); + + await rawKernel.connect(connectInfo); + + const code = 'print("hello world")'; + const executeContent: KernelMessage.IExecuteRequestMsg['content'] = { + code + }; + const future = rawKernel.requestExecute(executeContent, true, undefined); + + // Verify that we sent a message to jmp + verify(jmpConnection.sendMessage(anything())).once(); + + // We don't need a detailed verification on the jmp message sent, as that same + // message is set in the future which we can examine now + expect(future.msg.header.msg_type).to.equal('execute_request'); + expect(future.msg.channel).to.equal('shell'); + expect(future.msg.content.code).to.equal(code); + }); + + test('RawKernel dispose should also dispose of any futures', async () => { + when(jmpConnection.sendMessage(anything())).thenReturn(); + when(jmpConnection.dispose()).thenReturn(); + + await rawKernel.connect(connectInfo); + + const code = 'print("hello world")'; + const executeContent: KernelMessage.IExecuteRequestMsg['content'] = { + code + }; + const future = rawKernel.requestExecute(executeContent, true, undefined); + future.done.catch(reason => { + const error = reason as Error; + expect(error.message).to.equal('Disposed Future'); + }); + + // Dispose the rawKernel, the done promise on the future should reject with an Error + rawKernel.dispose(); + + expect(future.isDisposed).to.equal(true, 'Future was not disposed on RawKernel dispose'); + }); +}); diff --git a/src/test/linters/linterCommands.unit.test.ts b/src/test/linters/linterCommands.unit.test.ts index 3595f6e6837b..4f7259fd41d7 100644 --- a/src/test/linters/linterCommands.unit.test.ts +++ b/src/test/linters/linterCommands.unit.test.ts @@ -12,6 +12,7 @@ import { CommandManager } from '../../client/common/application/commandManager'; import { DocumentManager } from '../../client/common/application/documentManager'; import { IApplicationShell, ICommandManager, IDocumentManager } from '../../client/common/application/types'; import { Commands } from '../../client/common/constants'; +import { Product } from '../../client/common/types'; import { ServiceContainer } from '../../client/ioc/container'; import { LinterCommands } from '../../client/linters/linterCommands'; import { LinterManager } from '../../client/linters/linterManager'; @@ -163,6 +164,6 @@ suite('Linting - Linter Commands', () => { verify(shell.showWarningMessage(anything(), 'Yes', 'No')).once(); const quickPickOptions = capture(shell.showQuickPick).last()[1]; expect(quickPickOptions).to.deep.equal(expectedQuickPickOptions); - verify(manager.setActiveLintersAsync(deepEqual(['Three']), anything())).once(); + verify(manager.setActiveLintersAsync(deepEqual([('Three' as any) as Product]), anything())).once(); }); }); diff --git a/src/test/testing/pytest/services/discoveryService.unit.test.ts b/src/test/testing/pytest/services/discoveryService.unit.test.ts index fb2d71a7ffe5..8e9907a31a3a 100644 --- a/src/test/testing/pytest/services/discoveryService.unit.test.ts +++ b/src/test/testing/pytest/services/discoveryService.unit.test.ts @@ -91,11 +91,13 @@ suite('Unit Tests - PyTest - Discovery', () => { throw new Error('Unrecognized directory'); }; when(argsService.getTestFolders(deepEqual(options.args))).thenReturn(directories); - when(helper.mergeTests(deepEqual(['Result A', 'Result B']))).thenReturn('mergedTests' as any); + when(helper.mergeTests(deepEqual([('Result A' as any) as Tests, ('Result B' as any) as Tests]))).thenReturn( + 'mergedTests' as any + ); const tests = await discoveryService.discoverTests(options); - verify(helper.mergeTests(deepEqual(['Result A', 'Result B']))).once(); + verify(helper.mergeTests(deepEqual([('Result A' as any) as Tests, ('Result B' as any) as Tests]))).once(); expect(tests).equal('mergedTests'); }); test('Build collection arguments', async () => { From 686c5719aeb9752da0308c088f1c9780a62680ee Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Fri, 20 Mar 2020 17:21:25 -0700 Subject: [PATCH 003/725] Add preliminary support for zeromq v6 (#10682) * Preliminary idea building * Get test to actually start a kernel * Socket event emitter * Send working * Fix result messages * Add another comment to make sure to fix the potential stack issue * Fix hygiene and some code review comments. * resolve promise stack * Fix functional tests * More review feedback * Fix linter --- ThirdPartyNotices-Repository.txt | 33 ++ package-lock.json | 189 ++++++++---- package.json | 10 +- .../interactive-common/interactiveBase.ts | 3 +- .../datascience/jupyter/jupyterExecution.ts | 13 +- .../jupyter/kernels/kernelSelections.ts | 6 +- .../enchannel-zmq-backend-6/index.ts | 281 ++++++++++++++++++ .../raw-kernel/enchannelJMPConnection.ts | 47 +-- src/client/datascience/serviceRegistry.ts | 3 + src/client/datascience/types.ts | 2 + .../datascience/dataScienceIocContainer.ts | 3 + .../raw-kernel/rawKernel.functional.test.ts | 115 +++++++ 12 files changed, 607 insertions(+), 98 deletions(-) create mode 100644 src/client/datascience/raw-kernel/enchannel-zmq-backend-6/index.ts create mode 100644 src/test/datascience/raw-kernel/rawKernel.functional.test.ts diff --git a/ThirdPartyNotices-Repository.txt b/ThirdPartyNotices-Repository.txt index 21582af67bdf..80903f0ceeab 100644 --- a/ThirdPartyNotices-Repository.txt +++ b/ThirdPartyNotices-Repository.txt @@ -7,6 +7,7 @@ Microsoft Python extension for Visual Studio Code incorporates third party mater 1. Go for Visual Studio Code (https://github.com/Microsoft/vscode-go) 2. Files from the Python Project (https://www.python.org/) 3. Google Diff Match and Patch (https://github.com/GerHobbelt/google-diff-match-patch) +4. enchannel-zmq-backend (https://github.com/nteract/enchannel-zmq-backend) 6. omnisharp-vscode (https://github.com/OmniSharp/omnisharp-vscode) 8. PTVS (https://github.com/Microsoft/PTVS) 9. Python documentation (https://docs.python.org/) @@ -1093,3 +1094,35 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ========================================= END OF vscode-cpptools NOTICES, INFORMATION, AND LICENSE + +%% enchannel-zmq-backend NOTICES, INFORMATION, AND LICENSE BEGIN HERE +========================================= +Copyright (c) 2016, +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of enchannel-zmq-backend nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +========================================= +END OF enchannel-zmq-backend NOTICES, INFORMATION, AND LICENSE \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 1d8f357e0cd9..c4fc19eab3d9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,23 +5,49 @@ "requires": true, "dependencies": { "@babel/cli": { - "version": "7.5.0", - "resolved": "https://registry.npmjs.org/@babel/cli/-/cli-7.5.0.tgz", - "integrity": "sha512-qNH55fWbKrEsCwID+Qc/3JDPnsSGpIIiMDbppnR8Z6PxLAqMQCFNqBctkIkBrMH49Nx+qqVTrHRWUR+ho2k+qQ==", + "version": "7.8.4", + "resolved": "https://registry.npmjs.org/@babel/cli/-/cli-7.8.4.tgz", + "integrity": "sha512-XXLgAm6LBbaNxaGhMAznXXaxtCWfuv6PIDJ9Alsy9JYTOh+j2jJz+L/162kkfU1j/pTSxK1xGmlwI4pdIMkoag==", "dev": true, "requires": { - "chokidar": "^2.0.4", - "commander": "^2.8.1", + "chokidar": "^2.1.8", + "commander": "^4.0.1", "convert-source-map": "^1.1.0", "fs-readdir-recursive": "^1.1.0", "glob": "^7.0.0", - "lodash": "^4.17.11", - "mkdirp": "^0.5.1", - "output-file-sync": "^2.0.0", + "lodash": "^4.17.13", + "make-dir": "^2.1.0", "slash": "^2.0.0", "source-map": "^0.5.0" }, "dependencies": { + "chokidar": { + "version": "2.1.8", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-2.1.8.tgz", + "integrity": "sha512-ZmZUazfOzf0Nve7duiCKD23PFSCs4JPoYyccjUFF3aQkQadqBhfzhjkwBH2mNOG9cTBwhamM37EIsIkZw3nRgg==", + "dev": true, + "optional": true, + "requires": { + "anymatch": "^2.0.0", + "async-each": "^1.0.1", + "braces": "^2.3.2", + "fsevents": "^1.2.7", + "glob-parent": "^3.1.0", + "inherits": "^2.0.3", + "is-binary-path": "^1.0.0", + "is-glob": "^4.0.0", + "normalize-path": "^3.0.0", + "path-is-absolute": "^1.0.0", + "readdirp": "^2.2.1", + "upath": "^1.1.1" + } + }, + "commander": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", + "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", + "dev": true + }, "source-map": { "version": "0.5.7", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", @@ -895,24 +921,27 @@ } }, "@babel/register": { - "version": "7.4.4", - "resolved": "https://registry.npmjs.org/@babel/register/-/register-7.4.4.tgz", - "integrity": "sha512-sn51H88GRa00+ZoMqCVgOphmswG4b7mhf9VOB0LUBAieykq2GnRFerlN+JQkO/ntT7wz4jaHNSRPg9IdMPEUkA==", + "version": "7.8.6", + "resolved": "https://registry.npmjs.org/@babel/register/-/register-7.8.6.tgz", + "integrity": "sha512-7IDO93fuRsbyml7bAafBQb3RcBGlCpU4hh5wADA2LJEEcYk92WkwFZ0pHyIi2fb5Auoz1714abETdZKCOxN0CQ==", "dev": true, "requires": { - "core-js": "^3.0.0", "find-cache-dir": "^2.0.0", - "lodash": "^4.17.11", - "mkdirp": "^0.5.1", + "lodash": "^4.17.13", + "make-dir": "^2.1.0", "pirates": "^4.0.0", - "source-map-support": "^0.5.9" + "source-map-support": "^0.5.16" }, "dependencies": { - "core-js": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.1.4.tgz", - "integrity": "sha512-YNZN8lt82XIMLnLirj9MhKDFZHalwzzrL9YLt6eb0T5D0EDl4IQ90IGkua8mHbnxNrkj1d8hbdizMc0Qmg1WnQ==", - "dev": true + "source-map-support": { + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.16.tgz", + "integrity": "sha512-efyLRJDr68D9hBBNIPWFjhpFzURh+KJykQwvMyW5UiZzYwoF6l4YMMDIJJEyFWxWCqfyxLzz6tSfUFR+kXXsVQ==", + "dev": true, + "requires": { + "buffer-from": "^1.0.0", + "source-map": "^0.6.0" + } } } }, @@ -1141,9 +1170,9 @@ }, "dependencies": { "acorn": { - "version": "5.7.3", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.3.tgz", - "integrity": "sha512-T/zvzYRfbVojPWahDsE5evJdHb3oJoQfFbsrKM7w5Zcs++Tr257tia3BmMP8XYVjp1S9RZXQMh7gao96BlqZOw==", + "version": "5.7.4", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.4.tgz", + "integrity": "sha512-1D++VG7BhrtvQpNbBzovKNc1FLGGEE/oGe7b9xJm/RFHMBeUaUGpluV9RLjZa47YFdPcDAenEYuq9pQPcMdLJg==", "dev": true }, "normalize-path": { @@ -1429,6 +1458,15 @@ "tinyqueue": "^1.1.0" } }, + "@nteract/commutable": { + "version": "7.2.6", + "resolved": "https://registry.npmjs.org/@nteract/commutable/-/commutable-7.2.6.tgz", + "integrity": "sha512-pn2TVhnkrGumytmsmrKLhcKns1jBPdoeKnaihnSx1oaV55iUpfcWER9jgq/k7tcPronQy/dunJQid+s/Ch8mMQ==", + "requires": { + "immutable": "^4.0.0-rc.12", + "uuid": "^3.1.0" + } + }, "@nteract/markdown": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/@nteract/markdown/-/markdown-3.0.1.tgz", @@ -1451,6 +1489,28 @@ "babel-runtime": "^6.26.0" } }, + "@nteract/messaging": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/@nteract/messaging/-/messaging-7.0.0.tgz", + "integrity": "sha512-N/M8/7/seeSBYbtO0ZKzVeMnzJtVn5fiBSoDiEmvkFB8Y/UXoOeVxBusZVVZN054FbOgkX+vnRvaSEd75iRGag==", + "requires": { + "@nteract/types": "^6.0.0", + "@types/uuid": "^3.4.4", + "lodash.clonedeep": "^4.5.0", + "rxjs": "^6.3.3", + "uuid": "^3.1.0" + }, + "dependencies": { + "rxjs": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.4.tgz", + "integrity": "sha512-naMQXcgEo3csAEGvw/NydRA0fuS2nDZJiw1YUWFKU7aPPAPGZEsD4Iimit96qwCieH6y614MCLYwdkrWx7z/7Q==", + "requires": { + "tslib": "^1.9.0" + } + } + } + }, "@nteract/octicons": { "version": "0.5.1", "resolved": "https://registry.npmjs.org/@nteract/octicons/-/octicons-0.5.1.tgz", @@ -1590,6 +1650,27 @@ "react-json-tree": "^0.11.0" } }, + "@nteract/types": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/@nteract/types/-/types-6.0.0.tgz", + "integrity": "sha512-vpqZnXbJwQ31ngxHizs9c8t1g76Po3W4vay+zM5yiog8/bgWtfKiG8zVoQMesHrLGCJPt+QDoPIIXo2Ju61A3A==", + "requires": { + "@nteract/commutable": "^7.2.6", + "immutable": "^4.0.0-rc.12", + "rxjs": "^6.3.3", + "uuid": "^3.1.0" + }, + "dependencies": { + "rxjs": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.4.tgz", + "integrity": "sha512-naMQXcgEo3csAEGvw/NydRA0fuS2nDZJiw1YUWFKU7aPPAPGZEsD4Iimit96qwCieH6y614MCLYwdkrWx7z/7Q==", + "requires": { + "tslib": "^1.9.0" + } + } + } + }, "@nteract/vega-embed-v2": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@nteract/vega-embed-v2/-/vega-embed-v2-1.1.0.tgz", @@ -2385,8 +2466,7 @@ "@types/node": { "version": "10.14.18", "resolved": "https://registry.npmjs.org/@types/node/-/node-10.14.18.tgz", - "integrity": "sha512-ryO3Q3++yZC/+b8j8BdKd/dn9JlzlHBPdm80656xwYUdmPkpTGTjkAdt6BByiNupGPE8w0FhBgvYy/fX9hRNGQ==", - "dev": true + "integrity": "sha512-ryO3Q3++yZC/+b8j8BdKd/dn9JlzlHBPdm80656xwYUdmPkpTGTjkAdt6BByiNupGPE8w0FhBgvYy/fX9hRNGQ==" }, "@types/node-fetch": { "version": "2.3.7", @@ -2642,7 +2722,6 @@ "version": "3.4.5", "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-3.4.5.tgz", "integrity": "sha512-MNL15wC3EKyw1VLF+RoVO4hJJdk9t/Hlv3rt1OL65Qvuadm4BYo6g9ZJQqoq7X8NBFSsQXgAujWciovh2lpVjA==", - "dev": true, "requires": { "@types/node": "*" } @@ -9649,9 +9728,9 @@ }, "dependencies": { "acorn": { - "version": "5.7.3", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.3.tgz", - "integrity": "sha512-T/zvzYRfbVojPWahDsE5evJdHb3oJoQfFbsrKM7w5Zcs++Tr257tia3BmMP8XYVjp1S9RZXQMh7gao96BlqZOw==", + "version": "5.7.4", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-5.7.4.tgz", + "integrity": "sha512-1D++VG7BhrtvQpNbBzovKNc1FLGGEE/oGe7b9xJm/RFHMBeUaUGpluV9RLjZa47YFdPcDAenEYuq9pQPcMdLJg==", "dev": true } } @@ -10267,8 +10346,7 @@ "immutable": { "version": "4.0.0-rc.12", "resolved": "https://registry.npmjs.org/immutable/-/immutable-4.0.0-rc.12.tgz", - "integrity": "sha512-0M2XxkZLx/mi3t8NVwIm1g8nHoEmM9p9UBl/G9k4+hm0kBgOVdMV/B3CY5dQ8qG8qc80NN4gDV4HQv6FTJ5q7A==", - "dev": true + "integrity": "sha512-0M2XxkZLx/mi3t8NVwIm1g8nHoEmM9p9UBl/G9k4+hm0kBgOVdMV/B3CY5dQ8qG8qc80NN4gDV4HQv6FTJ5q7A==" }, "import-fresh": { "version": "2.0.0", @@ -11999,6 +12077,11 @@ "integrity": "sha1-soqmKIorn8ZRA1x3EfZathkDMaY=", "dev": true }, + "lodash.clonedeep": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz", + "integrity": "sha1-4j8/nE+Pvd6HJSnBBxhXoIblzO8=" + }, "lodash.curry": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/lodash.curry/-/lodash.curry-4.1.1.tgz", @@ -12600,9 +12683,9 @@ } }, "minimist": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", - "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" }, "minipass": { "version": "2.9.0", @@ -14431,17 +14514,6 @@ "os-tmpdir": "^1.0.0" } }, - "output-file-sync": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/output-file-sync/-/output-file-sync-2.0.1.tgz", - "integrity": "sha512-mDho4qm7WgIXIGf4eYU1RHN2UU5tPfVYVSRwDJw0uTmj35DQUt/eNp19N7v6T3SrR0ESTEf2up2CGO73qI35zQ==", - "dev": true, - "requires": { - "graceful-fs": "^4.1.11", - "is-plain-obj": "^1.1.0", - "mkdirp": "^0.5.1" - } - }, "p-cancelable": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-0.4.1.tgz", @@ -16695,13 +16767,18 @@ } }, "rxjs": { - "version": "5.5.12", - "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-5.5.12.tgz", - "integrity": "sha512-xx2itnL5sBbqeeiVgNPVuQQ1nC8Jp2WfNJhXWHmElW9YmrpS9UVnNzhP3EH3HFqexO5Tlp8GhYY+WEcqcVMvGw==", + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.4.tgz", + "integrity": "sha512-naMQXcgEo3csAEGvw/NydRA0fuS2nDZJiw1YUWFKU7aPPAPGZEsD4Iimit96qwCieH6y614MCLYwdkrWx7z/7Q==", "requires": { - "symbol-observable": "1.0.1" + "tslib": "^1.9.0" } }, + "rxjs-compat": { + "version": "6.5.4", + "resolved": "https://registry.npmjs.org/rxjs-compat/-/rxjs-compat-6.5.4.tgz", + "integrity": "sha512-rkn+lbOHUQOurdd74J/hjmDsG9nFx0z66fvnbs8M95nrtKvNqCKdk7iZqdY51CGmDemTQk+kUPy4s8HVOHtkfA==" + }, "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", @@ -18221,11 +18298,6 @@ "integrity": "sha512-7cXFbkZvPkZpKLC+3QIfyUd3/Un/CvJONjTD3Gz5qLuEa73StPOt8kZjTi9apxO6zwCaza0bPNnmzTyrQ4qQlw==", "dev": true }, - "symbol-observable": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-1.0.1.tgz", - "integrity": "sha1-g0D8RwLDEi310iKI+IKD9RPT/dQ=" - }, "symbol-tree": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", @@ -19141,8 +19213,7 @@ "tslib": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.10.0.tgz", - "integrity": "sha512-qOebF53frne81cf0S9B41ByenJ3/IuH8yJKngAX35CmiZySA0khhkovshKK+jGCaMnVomla7gVlIcc3EvKPbTQ==", - "dev": true + "integrity": "sha512-qOebF53frne81cf0S9B41ByenJ3/IuH8yJKngAX35CmiZySA0khhkovshKK+jGCaMnVomla7gVlIcc3EvKPbTQ==" }, "tslint": { "version": "5.20.1", @@ -19388,9 +19459,9 @@ } }, "typescript": { - "version": "3.7.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.7.2.tgz", - "integrity": "sha512-ml7V7JfiN2Xwvcer+XAf2csGO1bPBdRbFCkYBczNZggrBZ9c7G3riSUeJmqEU5uOtXNPMhE3n+R4FA/3YOAWOQ==", + "version": "3.8.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.8.3.tgz", + "integrity": "sha512-MYlEfn5VrLNsgudQTVJeNaQFUAI7DkhnOjdpAp4T+ku1TfQClewlbSuTVHiA+8skNBgaf02TL/kLOvig4y3G8w==", "dev": true }, "typescript-char": { diff --git a/package.json b/package.json index 3f64489147ed..88efde35687a 100644 --- a/package.json +++ b/package.json @@ -2905,6 +2905,7 @@ "@jupyterlab/services": "^4.2.0", "@koa/cors": "^3.0.0", "@loadable/component": "^5.12.0", + "@nteract/messaging": "^7.0.0", "ansi-regex": "^4.1.0", "arch": "^2.1.0", "azure-storage": "^2.10.3", @@ -2937,7 +2938,8 @@ "reflect-metadata": "^0.1.12", "request": "^2.87.0", "request-progress": "^3.0.0", - "rxjs": "^5.5.9", + "rxjs": "^6.5.4", + "rxjs-compat": "^6.5.4", "semver": "^5.5.0", "stack-trace": "0.0.10", "string-argv": "^0.3.1", @@ -2965,13 +2967,13 @@ "zeromq": "^6.0.0-beta.6" }, "devDependencies": { - "@babel/cli": "^7.4.4", + "@babel/cli": "^7.8.4", "@babel/core": "^7.4.4", "@babel/plugin-transform-runtime": "^7.4.4", "@babel/polyfill": "^7.4.4", "@babel/preset-env": "^7.1.0", "@babel/preset-react": "^7.0.0", - "@babel/register": "^7.4.4", + "@babel/register": "^7.8.6", "@blueprintjs/select": "^3.11.2", "@enonic/fnv-plus": "^1.3.0", "@istanbuljs/nyc-config-typescript": "^0.1.3", @@ -3142,7 +3144,7 @@ "tslint-plugin-prettier": "^2.1.0", "typed-react-markdown": "^0.1.0", "typemoq": "^2.1.0", - "typescript": "^3.7.2", + "typescript": "^3.8.3", "typescript-formatter": "^7.1.0", "unicode-properties": "^1.3.1", "url-loader": "^1.1.2", diff --git a/src/client/datascience/interactive-common/interactiveBase.ts b/src/client/datascience/interactive-common/interactiveBase.ts index 528dcf4adfa8..79e248a33ce6 100644 --- a/src/client/datascience/interactive-common/interactiveBase.ts +++ b/src/client/datascience/interactive-common/interactiveBase.ts @@ -589,6 +589,7 @@ export abstract class InteractiveBase extends WebViewHost c.state === CellState.error) === undefined; } }, diff --git a/src/client/datascience/jupyter/jupyterExecution.ts b/src/client/datascience/jupyter/jupyterExecution.ts index ce4d3492a631..7053a06c11b8 100644 --- a/src/client/datascience/jupyter/jupyterExecution.ts +++ b/src/client/datascience/jupyter/jupyterExecution.ts @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. 'use strict'; -import * as portfinder from 'portfinder'; import * as uuid from 'uuid/v4'; import { CancellationToken, CancellationTokenSource, Event, EventEmitter } from 'vscode'; @@ -9,7 +8,6 @@ import { IApplicationShell, ILiveShareApi, IWorkspaceService } from '../../commo import { Cancellation } from '../../common/cancellation'; import { traceError, traceInfo } from '../../common/logger'; import { IConfigurationService, IDisposableRegistry, IOutputChannel } from '../../common/types'; -import { sleep } from '../../common/utils/async'; import * as localize from '../../common/utils/localize'; import { noop } from '../../common/utils/misc'; import { StopWatch } from '../../common/utils/stopWatch'; @@ -339,15 +337,8 @@ export class JupyterExecutionBase implements IJupyterExecution { throw this.zmqError; } try { - const zmq = await import('zeromq'); - const sock = new zmq.Push(); - const port = await portfinder.getPortPromise(); - - await sock.bind(`tcp://127.0.0.1:${port}`); - sock.send('some work').ignoreErrors(); // This will never return unless there's a listener. Just used for testing the API is available - await sleep(50); - sock.close(); - traceInfo(`ZMQ connection to port ${port} verified.`); + await import('zeromq'); + traceInfo(`ZMQ install verified.`); } catch (e) { traceError(`Exception while attempting zmq :`, e); sendTelemetryEvent(Telemetry.ZMQNotSupported); diff --git a/src/client/datascience/jupyter/kernels/kernelSelections.ts b/src/client/datascience/jupyter/kernels/kernelSelections.ts index fc4f49dbb9ea..1951faed0788 100644 --- a/src/client/datascience/jupyter/kernels/kernelSelections.ts +++ b/src/client/datascience/jupyter/kernels/kernelSelections.ts @@ -112,8 +112,10 @@ export class InstalledJupyterKernelSelectionListProvider implements IKernelSelec ): Promise { const items = await this.kernelService.getKernelSpecs(this.sessionManager, cancelToken); return items - .filter(item => (item.language || '').toLowerCase() === PYTHON_LANGUAGE.toLowerCase()) - .map(item => getQuickPickItemForKernelSpec(item, this.pathUtils)); + ? items + .filter(item => (item.language || '').toLowerCase() === PYTHON_LANGUAGE.toLowerCase()) + .map(item => getQuickPickItemForKernelSpec(item, this.pathUtils)) + : []; } } diff --git a/src/client/datascience/raw-kernel/enchannel-zmq-backend-6/index.ts b/src/client/datascience/raw-kernel/enchannel-zmq-backend-6/index.ts new file mode 100644 index 000000000000..ba98b02a6774 --- /dev/null +++ b/src/client/datascience/raw-kernel/enchannel-zmq-backend-6/index.ts @@ -0,0 +1,281 @@ +// This code was copied from https://github.com/nteract/enchannel-zmq-backend/blob/master/src/index.ts +// and modified to work with zeromq-beta-6 + +import { Channels, JupyterMessage } from '@nteract/messaging'; +import * as wireProtocol from '@nteract/messaging/lib/wire-protocol'; +import * as Events from 'events'; +import * as rxjs from 'rxjs'; +import { map, publish, refCount } from 'rxjs/operators'; +import { v4 as uuid } from 'uuid'; +import * as zeromq from 'zeromq'; +import { traceError } from '../../../common/logger'; + +type ChannelName = 'iopub' | 'stdin' | 'shell' | 'control'; + +// tslint:disable: interface-name no-any +export interface JupyterConnectionInfo { + version: number; + iopub_port: number; + shell_port: number; + stdin_port: number; + control_port: number; + signature_scheme: 'hmac-sha256'; + hb_port: number; + ip: string; + key: string; + transport: 'tcp' | 'ipc'; +} + +interface HeaderFiller { + session: string; + username: string; +} + +/** + * Takes a Jupyter spec connection info object and channel and returns the + * string for a channel. Abstracts away tcp and ipc connection string + * formatting + * + * @param config Jupyter connection information + * @param channel Jupyter channel ("iopub", "shell", "control", "stdin") + * + * @returns The connection string + */ +export const formConnectionString = (config: JupyterConnectionInfo, channel: ChannelName) => { + const portDelimiter = config.transport === 'tcp' ? ':' : '-'; + const port = config[`${channel}_port` as keyof JupyterConnectionInfo]; + if (!port) { + throw new Error(`Port not found for channel "${channel}"`); + } + return `${config.transport}://${config.ip}${portDelimiter}${port}`; +}; + +/** + * Creates a socket for the given channel with ZMQ channel type given a config + * + * @param channel Jupyter channel ("iopub", "shell", "control", "stdin") + * @param identity UUID + * @param config Jupyter connection information + * + * @returns The new Jupyter ZMQ socket + */ +export async function createSubscriber( + channel: ChannelName, + config: JupyterConnectionInfo +): Promise { + const socket = new zeromq.Subscriber(); + + const url = formConnectionString(config, channel); + socket.connect(url); + return socket; +} + +/** + * Creates a socket for the given channel with ZMQ channel type given a config + * + * @param channel Jupyter channel ("iopub", "shell", "control", "stdin") + * @param identity UUID + * @param config Jupyter connection information + * + * @returns The new Jupyter ZMQ socket + */ +export async function createDealer( + channel: ChannelName, + identity: string, + config: JupyterConnectionInfo +): Promise { + // tslint:disable-next-line: no-require-imports + const socket = new zeromq.Dealer({ routingId: identity }); + + const url = formConnectionString(config, channel); + socket.connect(url); + return socket; +} + +export const getUsername = () => + process.env.LOGNAME || process.env.USER || process.env.LNAME || process.env.USERNAME || 'username'; // This is the fallback that the classic notebook uses + +interface Sockets { + shell: zeromq.Dealer; + control: zeromq.Dealer; + stdin: zeromq.Dealer; + iopub: zeromq.Subscriber; +} + +/** + * Sets up the sockets for each of the jupyter channels. + * + * @param config Jupyter connection information + * @param subscription The topic to filter the subscription to the iopub channel on + * @param identity UUID + * @param jmp A reference to the JMP Node module + * + * @returns Sockets for each Jupyter channel + */ +export const createSockets = async ( + config: JupyterConnectionInfo, + subscription: string = '', + identity = uuid() +): Promise => { + const [shell, control, stdin, iopub] = await Promise.all([ + createDealer('shell', identity, config), + createDealer('control', identity, config), + createDealer('stdin', identity, config), + createSubscriber('iopub', config) + ]); + + // NOTE: ZMQ PUB/SUB subscription (not an Rx subscription) + iopub.subscribe(subscription); + + return { + shell, + control, + stdin, + iopub + }; +}; + +class SocketEventEmitter extends Events.EventEmitter { + constructor(socket: zeromq.Dealer | zeromq.Subscriber) { + super(); + this.waitForReceive(socket); + } + + private waitForReceive(socket: zeromq.Dealer | zeromq.Subscriber) { + if (!socket.closed) { + // tslint:disable-next-line: no-floating-promises + socket + .receive() + .then(b => { + this.emit('message', b); + setTimeout(this.waitForReceive.bind(this, socket), 0); + }) + .ignoreErrors(); + } + } +} + +/** + * Creates a multiplexed set of channels. + * + * @param sockets An object containing associations between channel types and 0MQ sockets + * @param header The session and username to place in kernel message headers + * @param jmp A reference to the JMP Node module + * + * @returns Creates an Observable for each channel connection that allows us + * to send and receive messages through the Jupyter protocol. + */ +export const createMainChannelFromSockets = ( + sockets: Sockets, + connectionInfo: JupyterConnectionInfo, + header: HeaderFiller = { + session: uuid(), + username: getUsername() + } +): Channels => { + // The mega subject that encapsulates all the sockets as one multiplexed + // stream + const outgoingMessages = rxjs.Subscriber.create( + async message => { + // There's always a chance that a bad message is sent, we'll ignore it + // instead of consuming it + if (!message || !message.channel) { + console.warn('message sent without a channel', message); + return; + } + const socket = (sockets as any)[message.channel]; + if (!socket) { + // If, for some reason, a message is sent on a channel we don't have + // a socket for, warn about it but don't bomb the stream + console.warn('channel not understood for message', message); + return; + } + try { + const jMessage: wireProtocol.RawJupyterMessage = { + // Fold in the setup header to ease usage of messages on channels + header: { ...message.header, ...header }, + parent_header: message.parent_header as any, + content: message.content, + metadata: message.metadata, + buffers: message.buffers as any, + idents: [] + }; + if ((socket as any).send !== undefined) { + await (socket as zeromq.Dealer).send( + wireProtocol.encode(jMessage, connectionInfo.key, connectionInfo.signature_scheme) + ); + } + } catch (err) { + traceError('Error sending message', err, message); + } + }, + undefined, // not bothering with sending errors on + () => { + // When the subject is completed / disposed, close all the event + // listeners and shutdown the socket + const closer = (closable: { close(): void }) => { + try { + closable.close(); + } catch (ex) { + traceError(`Error during socket shutdown`, ex); + } + }; + closer(sockets.control); + closer(sockets.iopub); + closer(sockets.shell); + closer(sockets.stdin); + } + ); + + // Messages from kernel on the sockets + const incomingMessages: rxjs.Observable = rxjs + .merge( + // Form an Observable with each socket + ...Object.keys(sockets).map(name => { + // Wrap in something that will emit an event whenever a message is received. + const socketEmitter = new SocketEventEmitter((sockets as any)[name]); + return rxjs.fromEvent(socketEmitter, 'message').pipe( + map( + (body: any): JupyterMessage => { + return wireProtocol.decode( + body, + connectionInfo.key, + connectionInfo.signature_scheme + ) as any; + } + ), + publish(), + refCount() + ); + }) + ) + .pipe(publish(), refCount()); + + return rxjs.Subject.create(outgoingMessages, incomingMessages); +}; + +/** + * Creates a multiplexed set of channels. + * + * @param config Jupyter connection information + * @param config.ip IP address of the kernel + * @param config.transport Transport, e.g. TCP + * @param config.signature_scheme Hashing scheme, e.g. hmac-sha256 + * @param config.iopub_port Port for iopub channel + * @param subscription subscribed topic; defaults to all + * @param identity UUID + * + * @returns Subject containing multiplexed channels + */ +export const createMainChannel = async ( + config: JupyterConnectionInfo, + subscription: string = '', + identity: string = uuid(), + header: HeaderFiller = { + session: uuid(), + username: getUsername() + } +): Promise => { + const sockets = await createSockets(config, subscription, identity); + return createMainChannelFromSockets(sockets, config, header); +}; diff --git a/src/client/datascience/raw-kernel/enchannelJMPConnection.ts b/src/client/datascience/raw-kernel/enchannelJMPConnection.ts index 14f0d238b7af..06c6b029b6ed 100644 --- a/src/client/datascience/raw-kernel/enchannelJMPConnection.ts +++ b/src/client/datascience/raw-kernel/enchannelJMPConnection.ts @@ -1,34 +1,39 @@ import { KernelMessage } from '@jupyterlab/services'; -//import { Channels } from '@nteract/messaging'; -//import { createMainChannel } from 'enchannel-zmq-backend'; +import { Channels } from '@nteract/messaging'; +import { injectable } from 'inversify'; import { IJMPConnection, IJMPConnectionInfo } from '../types'; +@injectable() export class EnchannelJMPConnection implements IJMPConnection { - //private mainChannel: Channels | undefined; + private mainChannel: Channels | undefined; + + public async connect(connectInfo: IJMPConnectionInfo, sessionID: string): Promise { + // zmq may not load, so do it dynamically + // tslint:disable-next-line: no-require-imports + const enchannelZmq6 = (await require('./enchannel-zmq-backend-6/index')) as typeof import('./enchannel-zmq-backend-6/index'); - public async connect(_connectInfo: IJMPConnectionInfo, _sessionID: string): Promise { // tslint:disable-next-line:no-any - //this.mainChannel = await createMainChannel(connectInfo as any, undefined, sessionID); + this.mainChannel = await enchannelZmq6.createMainChannel(connectInfo as any, undefined, sessionID); } - public sendMessage(_message: KernelMessage.IMessage): void { - //if (this.mainChannel) { - //// jupyterlab types and enchannel types seem to have small changes - //// with how they are defined, just use an any cast for now, but they appear to be the - //// same actual object - //// tslint:disable-next-line:no-any - //this.mainChannel.next(message as any); - //} + public sendMessage(message: KernelMessage.IMessage): void { + if (this.mainChannel) { + // jupyterlab types and enchannel types seem to have small changes + // with how they are defined, just use an any cast for now, but they appear to be the + // same actual object + // tslint:disable-next-line:no-any + this.mainChannel.next(message as any); + } } - public subscribe(_handlerFunc: (message: KernelMessage.IMessage) => void) { - //if (this.mainChannel) { - //// tslint:disable-next-line:no-any - //this.mainChannel.subscribe(handlerFunc as any); - //} + public subscribe(handlerFunc: (message: KernelMessage.IMessage) => void) { + if (this.mainChannel) { + // tslint:disable-next-line:no-any + this.mainChannel.subscribe(handlerFunc as any); + } } public dispose(): void { - //if (this.mainChannel) { - //this.mainChannel.unsubscribe(); - //} + if (this.mainChannel) { + this.mainChannel.unsubscribe(); + } } } diff --git a/src/client/datascience/serviceRegistry.ts b/src/client/datascience/serviceRegistry.ts index c0e0002a9f2e..b518e925afa8 100644 --- a/src/client/datascience/serviceRegistry.ts +++ b/src/client/datascience/serviceRegistry.ts @@ -76,6 +76,7 @@ import { PlotViewer } from './plotting/plotViewer'; import { PlotViewerProvider } from './plotting/plotViewerProvider'; import { PreWarmActivatedJupyterEnvironmentVariables } from './preWarmVariables'; import { ProgressReporter } from './progress/progressReporter'; +import { EnchannelJMPConnection } from './raw-kernel/enchannelJMPConnection'; import { StatusProvider } from './statusProvider'; import { ThemeFinder } from './themeFinder'; import { @@ -97,6 +98,7 @@ import { IInteractiveWindow, IInteractiveWindowListener, IInteractiveWindowProvider, + IJMPConnection, IJupyterCommandFactory, IJupyterDebugger, IJupyterExecution, @@ -193,6 +195,7 @@ export function registerTypes(serviceManager: IServiceManager) { serviceManager.addSingleton(NativeEditorSynchronizer, NativeEditorSynchronizer); serviceManager.addSingleton(InteractiveWindowNotebookProvider, InteractiveWindowNotebookProvider); serviceManager.addSingleton(NativeNotebookProvider, NativeNotebookProvider); + serviceManager.add(IJMPConnection, EnchannelJMPConnection); // Temporary code, to allow users to revert to the old behavior. const cfg = serviceManager.get(IWorkspaceService).getConfiguration('python.dataScience', undefined); diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index fb289ada56cb..1d1c2cb071cf 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -872,6 +872,8 @@ export interface IJMPConnectionInfo { transport: string; } +export const IJMPConnection = Symbol('IJMPConnection'); + // A service to send and recieve messages over Jupyter messaging protocol export interface IJMPConnection extends IDisposable { connect(connectInfo: IJMPConnectionInfo, sessionID: string): Promise; diff --git a/src/test/datascience/dataScienceIocContainer.ts b/src/test/datascience/dataScienceIocContainer.ts index 6a2693e08211..9f897d735352 100644 --- a/src/test/datascience/dataScienceIocContainer.ts +++ b/src/test/datascience/dataScienceIocContainer.ts @@ -216,6 +216,7 @@ import { JupyterServerSelector } from '../../client/datascience/jupyter/serverSe import { PlotViewer } from '../../client/datascience/plotting/plotViewer'; import { PlotViewerProvider } from '../../client/datascience/plotting/plotViewerProvider'; import { ProgressReporter } from '../../client/datascience/progress/progressReporter'; +import { EnchannelJMPConnection } from '../../client/datascience/raw-kernel/enchannelJMPConnection'; import { StatusProvider } from '../../client/datascience/statusProvider'; import { ThemeFinder } from '../../client/datascience/themeFinder'; import { @@ -237,6 +238,7 @@ import { IInteractiveWindow, IInteractiveWindowListener, IInteractiveWindowProvider, + IJMPConnection, IJupyterCommandFactory, IJupyterDebugger, IJupyterExecution, @@ -1030,6 +1032,7 @@ export class DataScienceIocContainer extends UnitTestIocContainer { this.serviceManager.addSingleton(KernelService, KernelService); this.serviceManager.addSingleton(IProcessServiceFactory, ProcessServiceFactory); this.serviceManager.addSingleton(IPythonExecutionFactory, PythonExecutionFactory); + this.serviceManager.add(IJMPConnection, EnchannelJMPConnection); // Make sure full interpreter services are available. registerInterpreterTypes(this.serviceManager); diff --git a/src/test/datascience/raw-kernel/rawKernel.functional.test.ts b/src/test/datascience/raw-kernel/rawKernel.functional.test.ts new file mode 100644 index 000000000000..68b5b08e0f32 --- /dev/null +++ b/src/test/datascience/raw-kernel/rawKernel.functional.test.ts @@ -0,0 +1,115 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +'use strict'; +import { KernelMessage } from '@jupyterlab/services'; +import { assert } from 'chai'; +import * as fs from 'fs-extra'; +import { noop } from 'jquery'; +import * as os from 'os'; +import * as path from 'path'; +import * as uuid from 'uuid/v4'; +import { IPythonExecutionFactory, ObservableExecutionResult } from '../../../client/common/process/types'; +import { createDeferred } from '../../../client/common/utils/async'; +import { IJMPConnection } from '../../../client/datascience/types'; +import { DataScienceIocContainer } from '../dataScienceIocContainer'; + +// tslint:disable:no-any no-multiline-string max-func-body-length no-console max-classes-per-file trailing-comma +suite('DataScience raw kernel tests', () => { + let ioc: DataScienceIocContainer; + let enchannelConnection: IJMPConnection; + let connectionFile: string; + let kernelProcResult: ObservableExecutionResult; + const connectionInfo = { + shell_port: 57718, + iopub_port: 57719, + stdin_port: 57720, + control_port: 57721, + hb_port: 57722, + ip: '127.0.0.1', + key: 'c29c2121-d277576c2c035f0aceeb5068', + transport: 'tcp', + signature_scheme: 'hmac-sha256', + kernel_name: 'python3', + version: 5.1 + }; + setup(async function() { + ioc = new DataScienceIocContainer(); + ioc.registerDataScienceTypes(); + await ioc.activate(); + if (ioc.mockJupyter) { + // tslint:disable-next-line: no-invalid-this + this.skip(); + } else { + enchannelConnection = ioc.get(IJMPConnection); + + // Find our jupyter interpreter + const interpreter = await ioc.getJupyterCapableInterpreter(); + assert.ok(interpreter, 'No jupyter interpreter found'); + // Start our kernel + const execFactory = ioc.get(IPythonExecutionFactory); + const env = await execFactory.createActivatedEnvironment({ interpreter }); + + connectionFile = path.join(os.tmpdir(), `tmp_${Date.now()}_k.json`); + await fs.writeFile(connectionFile, JSON.stringify(connectionInfo), { encoding: 'utf-8', flag: 'w' }); + + // Keep kernel alive while the tests are running. + kernelProcResult = env.execObservable(['-m', 'ipykernel_launcher', '-f', connectionFile], { + throwOnStdErr: false + }); + kernelProcResult.out.subscribe( + out => { + console.log(out.out); + }, + error => { + console.error(error); + }, + () => { + enchannelConnection.dispose(); + } + ); + } + }); + + teardown(async () => { + kernelProcResult?.proc?.kill(); + try { + await fs.remove(connectionFile); + } catch { + noop(); + } + await ioc.dispose(); + }); + + function createShutdownMessage(sessionId: string): KernelMessage.IMessage<'shutdown_request'> { + return { + channel: 'control', + content: { + restart: false + }, + header: { + date: Date.now().toString(), + msg_id: uuid(), + msg_type: 'shutdown_request', + session: sessionId, + username: 'user', + version: '5.1' + }, + parent_header: {}, + metadata: {} + }; + } + + // tslint:disable-next-line: no-function-expression + test('Basic iopub', async function() { + const sessionId = uuid(); + const reply = createDeferred(); + await enchannelConnection.connect(connectionInfo, sessionId); + enchannelConnection.subscribe(msg => { + if (msg.header.msg_type === 'status') { + reply.resolve(); + } + }); + enchannelConnection.sendMessage(createShutdownMessage(sessionId)); + await reply.promise; + }); +}); From 0f732ca185134bcd6689f132ebe674f833884c40 Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Mon, 23 Mar 2020 09:35:29 -0700 Subject: [PATCH 004/725] Add status, fix message ordering, added full executeRequest unit test (#10698) --- .../raw-kernel/enchannelJMPConnection.ts | 4 +- .../datascience/raw-kernel/rawKernel.ts | 65 ++++- src/client/datascience/types.ts | 2 +- src/test/datascience/raw-kernel/mockJMP.ts | 28 ++ .../raw-kernel/rawKernel.functional.test.ts | 5 +- .../raw-kernel/rawKernel.unit.test.ts | 258 +++++++++++++----- 6 files changed, 279 insertions(+), 83 deletions(-) create mode 100644 src/test/datascience/raw-kernel/mockJMP.ts diff --git a/src/client/datascience/raw-kernel/enchannelJMPConnection.ts b/src/client/datascience/raw-kernel/enchannelJMPConnection.ts index 06c6b029b6ed..60bda1808b1c 100644 --- a/src/client/datascience/raw-kernel/enchannelJMPConnection.ts +++ b/src/client/datascience/raw-kernel/enchannelJMPConnection.ts @@ -7,13 +7,13 @@ import { IJMPConnection, IJMPConnectionInfo } from '../types'; export class EnchannelJMPConnection implements IJMPConnection { private mainChannel: Channels | undefined; - public async connect(connectInfo: IJMPConnectionInfo, sessionID: string): Promise { + public async connect(connectInfo: IJMPConnectionInfo): Promise { // zmq may not load, so do it dynamically // tslint:disable-next-line: no-require-imports const enchannelZmq6 = (await require('./enchannel-zmq-backend-6/index')) as typeof import('./enchannel-zmq-backend-6/index'); // tslint:disable-next-line:no-any - this.mainChannel = await enchannelZmq6.createMainChannel(connectInfo as any, undefined, sessionID); + this.mainChannel = await enchannelZmq6.createMainChannel(connectInfo as any); } public sendMessage(message: KernelMessage.IMessage): void { if (this.mainChannel) { diff --git a/src/client/datascience/raw-kernel/rawKernel.ts b/src/client/datascience/raw-kernel/rawKernel.ts index 5e204b637f72..00720d8d641f 100644 --- a/src/client/datascience/raw-kernel/rawKernel.ts +++ b/src/client/datascience/raw-kernel/rawKernel.ts @@ -2,8 +2,9 @@ // Licensed under the MIT License. import { Kernel, KernelMessage, ServerConnection } from '@jupyterlab/services'; import { JSONObject } from '@phosphor/coreutils'; -import { ISignal } from '@phosphor/signaling'; +import { ISignal, Signal } from '@phosphor/signaling'; import * as uuid from 'uuid/v4'; +import { traceError } from '../../common/logger'; import { IJMPConnection, IJMPConnectionInfo } from '../types'; import { RawFuture } from './rawFuture'; @@ -18,7 +19,7 @@ export class RawKernel implements Kernel.IKernel { throw new Error('Not yet implemented'); } get statusChanged(): ISignal { - throw new Error('Not yet implemented'); + return this._statusChanged; } get iopubMessage(): ISignal { throw new Error('Not yet implemented'); @@ -47,10 +48,10 @@ export class RawKernel implements Kernel.IKernel { throw new Error('Not yet implemented'); } get clientId(): string { - throw new Error('Not yet implemented'); + return this._clientId; } get status(): Kernel.Status { - throw new Error('Not yet implemented'); + return this._status; } get info(): KernelMessage.IInfoReply | null { throw new Error('Not yet implemented'); @@ -67,7 +68,11 @@ export class RawKernel implements Kernel.IKernel { public isDisposed: boolean = false; private jmpConnection: IJMPConnection; - private sessionId: string | undefined; + private messageChain: Promise = Promise.resolve(); + + private _clientId: string; + private _status: Kernel.Status; + private _statusChanged: Signal; // Keep track of all of our active futures private futures = new Map< @@ -77,14 +82,16 @@ export class RawKernel implements Kernel.IKernel { // JMP connection should be injected, but no need to yet until it actually exists constructor(connection: IJMPConnection) { + this._clientId = uuid(); + this._status = 'unknown'; + this._statusChanged = new Signal(this); this.jmpConnection = connection; } public async connect(connectInfo: IJMPConnectionInfo) { - this.sessionId = uuid(); - await this.jmpConnection.connect(connectInfo, this.sessionId); - this.jmpConnection.subscribe(msg => { - this.handleMessage(msg).ignoreErrors(); + await this.jmpConnection.connect(connectInfo); + this.jmpConnection.subscribe(message => { + this.msgIn(message); }); } @@ -93,12 +100,12 @@ export class RawKernel implements Kernel.IKernel { disposeOnDone?: boolean, _metadata?: JSONObject ): Kernel.IShellFuture { - if (this.jmpConnection && this.sessionId) { + if (this.jmpConnection) { // Build our execution message // Silent is supposed to be options, but in my testing the message was not passing // correctly without it, so specifying it here with default false const executeOptions: KernelMessage.IOptions = { - session: this.sessionId, + session: this._clientId, channel: 'shell', msgType: 'execute_request', username: 'vscode', @@ -244,6 +251,21 @@ export class RawKernel implements Kernel.IKernel { throw new Error('Not yet implemented'); } + // Message incoming from the JMP connection. Queue it up for processing + private msgIn(message: KernelMessage.IMessage) { + // Add the message onto our message chain, we want to process them async + // but in order so use a chain like this + this.messageChain = this.messageChain + .then(() => { + // Return so any promises from each message all resolve before + // processing the next one + return this.handleMessage(message); + }) + .catch(error => { + traceError(error); + }); + } + // Handle a new message arriving from JMP connection private async handleMessage(message: KernelMessage.IMessage): Promise { // RAWKERNEL: display_data messages can route based on their id here first @@ -257,12 +279,29 @@ export class RawKernel implements Kernel.IKernel { // Let the parent future message handle it here await parentFuture.handleMessage(message); } else { - if (message.header.session === this.sessionId && message.channel !== 'iopub') { + if (message.header.session === this._clientId && message.channel !== 'iopub') { // RAWKERNEL: emit unhandled } } } - // RAWKERNEL: Handle general IOpub messages + // Check for ioPub status messages + if (message.channel === 'iopub' && message.header.msg_type === 'status') { + const newStatus = (message as KernelMessage.IStatusMsg).content.execution_state; + this.updateStatus(newStatus); + } + } + + // The status for our kernel has changed + private updateStatus(newStatus: Kernel.Status) { + if (this._status === newStatus || this._status === 'dead') { + return; + } + + this._status = newStatus; + this._statusChanged.emit(newStatus); + if (newStatus === 'dead') { + this.dispose(); + } } } diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index 1d1c2cb071cf..03bdfc6e783d 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -876,7 +876,7 @@ export const IJMPConnection = Symbol('IJMPConnection'); // A service to send and recieve messages over Jupyter messaging protocol export interface IJMPConnection extends IDisposable { - connect(connectInfo: IJMPConnectionInfo, sessionID: string): Promise; + connect(connectInfo: IJMPConnectionInfo): Promise; sendMessage(message: KernelMessage.IMessage): void; subscribe(handlerFunc: (message: KernelMessage.IMessage) => void): void; } diff --git a/src/test/datascience/raw-kernel/mockJMP.ts b/src/test/datascience/raw-kernel/mockJMP.ts new file mode 100644 index 000000000000..7f530cbea907 --- /dev/null +++ b/src/test/datascience/raw-kernel/mockJMP.ts @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +import { KernelMessage } from '@jupyterlab/services'; +import { IJMPConnection, IJMPConnectionInfo } from '../../../client/datascience/types'; + +export class MockJMPConnection implements IJMPConnection { + private callback: ((message: KernelMessage.IMessage) => void) | undefined; + + public async connect(_connectInfo: IJMPConnectionInfo): Promise { + return; + } + public sendMessage(_message: KernelMessage.IMessage): void { + return; + } + public subscribe(handlerFunc: (message: KernelMessage.IMessage) => void): void { + this.callback = handlerFunc; + } + public dispose(): void { + return; + } + + // Send a kernel message back to the hander function + public messageBack(message: KernelMessage.IMessage) { + if (this.callback) { + this.callback(message); + } + } +} diff --git a/src/test/datascience/raw-kernel/rawKernel.functional.test.ts b/src/test/datascience/raw-kernel/rawKernel.functional.test.ts index 68b5b08e0f32..896d7824fa8f 100644 --- a/src/test/datascience/raw-kernel/rawKernel.functional.test.ts +++ b/src/test/datascience/raw-kernel/rawKernel.functional.test.ts @@ -101,15 +101,14 @@ suite('DataScience raw kernel tests', () => { // tslint:disable-next-line: no-function-expression test('Basic iopub', async function() { - const sessionId = uuid(); const reply = createDeferred(); - await enchannelConnection.connect(connectionInfo, sessionId); + await enchannelConnection.connect(connectionInfo); enchannelConnection.subscribe(msg => { if (msg.header.msg_type === 'status') { reply.resolve(); } }); - enchannelConnection.sendMessage(createShutdownMessage(sessionId)); + enchannelConnection.sendMessage(createShutdownMessage(uuid())); await reply.promise; }); }); diff --git a/src/test/datascience/raw-kernel/rawKernel.unit.test.ts b/src/test/datascience/raw-kernel/rawKernel.unit.test.ts index 2489eebf1ecc..16670d744d69 100644 --- a/src/test/datascience/raw-kernel/rawKernel.unit.test.ts +++ b/src/test/datascience/raw-kernel/rawKernel.unit.test.ts @@ -1,10 +1,12 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -import { KernelMessage } from '@jupyterlab/services'; +import { Kernel, KernelMessage } from '@jupyterlab/services'; +import { Slot } from '@phosphor/signaling'; import { assert, expect } from 'chai'; import { anything, deepEqual, instance, mock, verify, when } from 'ts-mockito'; import { RawKernel } from '../../../client/datascience/raw-kernel/rawKernel'; import { IJMPConnection, IJMPConnectionInfo } from '../../../client/datascience/types'; +import { MockJMPConnection } from './mockJMP'; // tslint:disable: max-func-body-length suite('Data Science - RawKernel', () => { @@ -12,84 +14,212 @@ suite('Data Science - RawKernel', () => { let jmpConnection: IJMPConnection; let connectInfo: IJMPConnectionInfo; - setup(() => { - jmpConnection = mock(); - when(jmpConnection.connect(anything(), anything())).thenResolve(); - when(jmpConnection.subscribe(anything())).thenReturn(); - rawKernel = new RawKernel(instance(jmpConnection)); - - connectInfo = { - version: 0, - transport: 'tcp', - ip: '127.0.0.1', - shell_port: 55196, - iopub_port: 55197, - stdin_port: 55198, - hb_port: 55200, - control_port: 55199, - signature_scheme: 'hmac-sha256', - key: 'adaf9032-487d222a85026db284c3d5e7' - }; - }); + suite('RawKernel basic mock jmp', () => { + setup(() => { + jmpConnection = mock(); + when(jmpConnection.connect(anything())).thenResolve(); + when(jmpConnection.subscribe(anything())).thenReturn(); + rawKernel = new RawKernel(instance(jmpConnection)); + + connectInfo = { + version: 0, + transport: 'tcp', + ip: '127.0.0.1', + shell_port: 55196, + iopub_port: 55197, + stdin_port: 55198, + hb_port: 55200, + control_port: 55199, + signature_scheme: 'hmac-sha256', + key: 'adaf9032-487d222a85026db284c3d5e7' + }; + }); - test('RawKernel connect should connect and subscribe to JMP', async () => { - await rawKernel.connect(connectInfo); - verify(jmpConnection.connect(deepEqual(connectInfo), anything())).once(); - verify(jmpConnection.subscribe(anything())).once(); - }); + test('RawKernel connect should connect and subscribe to JMP', async () => { + await rawKernel.connect(connectInfo); + verify(jmpConnection.connect(deepEqual(connectInfo))).once(); + verify(jmpConnection.subscribe(anything())).once(); + }); - test('RawKernel dispose should dispose the jmp', async () => { - when(jmpConnection.dispose()).thenReturn(); + test('RawKernel dispose should dispose the jmp', async () => { + when(jmpConnection.dispose()).thenReturn(); - await rawKernel.connect(connectInfo); + await rawKernel.connect(connectInfo); - // Dispose our kernel - rawKernel.dispose(); + // Dispose our kernel + rawKernel.dispose(); - verify(jmpConnection.dispose()).once(); - assert.isTrue(rawKernel.isDisposed); - }); + verify(jmpConnection.dispose()).once(); + assert.isTrue(rawKernel.isDisposed); + }); - test('RawKernel requestExecute should pass a valid execute message to JMP', async () => { - when(jmpConnection.sendMessage(anything())).thenReturn(); + test('RawKernel requestExecute should pass a valid execute message to JMP', async () => { + when(jmpConnection.sendMessage(anything())).thenReturn(); - await rawKernel.connect(connectInfo); + await rawKernel.connect(connectInfo); - const code = 'print("hello world")'; - const executeContent: KernelMessage.IExecuteRequestMsg['content'] = { - code - }; - const future = rawKernel.requestExecute(executeContent, true, undefined); + const code = 'print("hello world")'; + const executeContent: KernelMessage.IExecuteRequestMsg['content'] = { + code + }; + const future = rawKernel.requestExecute(executeContent, true, undefined); - // Verify that we sent a message to jmp - verify(jmpConnection.sendMessage(anything())).once(); + // Verify that we sent a message to jmp + verify(jmpConnection.sendMessage(anything())).once(); - // We don't need a detailed verification on the jmp message sent, as that same - // message is set in the future which we can examine now - expect(future.msg.header.msg_type).to.equal('execute_request'); - expect(future.msg.channel).to.equal('shell'); - expect(future.msg.content.code).to.equal(code); - }); + // We don't need a detailed verification on the jmp message sent, as that same + // message is set in the future which we can examine now + expect(future.msg.header.msg_type).to.equal('execute_request'); + expect(future.msg.channel).to.equal('shell'); + expect(future.msg.content.code).to.equal(code); + }); + + test('RawKernel dispose should also dispose of any futures', async () => { + when(jmpConnection.sendMessage(anything())).thenReturn(); + when(jmpConnection.dispose()).thenReturn(); - test('RawKernel dispose should also dispose of any futures', async () => { - when(jmpConnection.sendMessage(anything())).thenReturn(); - when(jmpConnection.dispose()).thenReturn(); + await rawKernel.connect(connectInfo); - await rawKernel.connect(connectInfo); + const code = 'print("hello world")'; + const executeContent: KernelMessage.IExecuteRequestMsg['content'] = { + code + }; + const future = rawKernel.requestExecute(executeContent, true, undefined); + future.done.catch(reason => { + const error = reason as Error; + expect(error.message).to.equal('Disposed Future'); + }); - const code = 'print("hello world")'; - const executeContent: KernelMessage.IExecuteRequestMsg['content'] = { - code - }; - const future = rawKernel.requestExecute(executeContent, true, undefined); - future.done.catch(reason => { - const error = reason as Error; - expect(error.message).to.equal('Disposed Future'); + // Dispose the rawKernel, the done promise on the future should reject with an Error + rawKernel.dispose(); + + expect(future.isDisposed).to.equal(true, 'Future was not disposed on RawKernel dispose'); }); + }); - // Dispose the rawKernel, the done promise on the future should reject with an Error - rawKernel.dispose(); + // These suite of tests need to use a mock jmp connection to send back messages as needed + suite('RawKernel advanced mock jmp', () => { + let mockJmpConnection: MockJMPConnection; - expect(future.isDisposed).to.equal(true, 'Future was not disposed on RawKernel dispose'); + setup(() => { + mockJmpConnection = new MockJMPConnection(); + rawKernel = new RawKernel(mockJmpConnection); + }); + + test('RawKernel executeRequest messages', async () => { + await rawKernel.connect(connectInfo); + + // Check our status at the start + expect(rawKernel.status).to.equal('unknown'); + + // Create a future for an execute code request + const code = 'print("hello world")'; + const executeContent: KernelMessage.IExecuteRequestMsg['content'] = { + code + }; + const future = rawKernel.requestExecute(executeContent, true, undefined); + + // First message is iopub busy status + const iopubBusyOptions: KernelMessage.IOptions = { + channel: 'iopub', + session: rawKernel.clientId, + msgType: 'status', + content: { execution_state: 'busy' } + }; + const iopubBusyMessage = KernelMessage.createMessage(iopubBusyOptions); + iopubBusyMessage.parent_header = future.msg.header; + + // Post the message + mockJmpConnection.messageBack(iopubBusyMessage); + + // Next iopub execute input + const iopubExecuteInputOptions: KernelMessage.IOptions = { + channel: 'iopub', + session: rawKernel.clientId, + msgType: 'execute_input', + content: { code, execution_count: 1 } + }; + const iopubExecuteInputMessage = KernelMessage.createMessage( + iopubExecuteInputOptions + ); + iopubExecuteInputMessage.parent_header = future.msg.header; + + // Post the message + mockJmpConnection.messageBack(iopubExecuteInputMessage); + + // Next iopub stream input + const iopubStreamOptions: KernelMessage.IOptions = { + channel: 'iopub', + session: rawKernel.clientId, + msgType: 'stream', + content: { name: 'stdout', text: 'hello' } + }; + const iopubStreamMessage = KernelMessage.createMessage(iopubStreamOptions); + iopubStreamMessage.parent_header = future.msg.header; + + // Post the message + mockJmpConnection.messageBack(iopubStreamMessage); + + // Finally an idle message + const iopubIdleOptions: KernelMessage.IOptions = { + channel: 'iopub', + session: rawKernel.clientId, + msgType: 'status', + content: { execution_state: 'idle' } + }; + const iopubIdleMessage = KernelMessage.createMessage(iopubIdleOptions); + iopubIdleMessage.parent_header = future.msg.header; + + // Post the message + mockJmpConnection.messageBack(iopubIdleMessage); + + // Last thing back is a reply message + const replyOptions: KernelMessage.IOptions = { + channel: 'shell', + session: rawKernel.clientId, + msgType: 'execute_reply', + content: { status: 'ok', execution_count: 1, payload: [], user_expressions: {} } + }; + const replyMessage = KernelMessage.createMessage(replyOptions); + replyMessage.parent_header = future.msg.header; + + mockJmpConnection.messageBack(replyMessage); + + // Before we await for done we need to set up what we expect to see in our output + + // Check our IOPub Messages + const iopubMessages = [iopubBusyMessage, iopubExecuteInputMessage, iopubStreamMessage, iopubIdleMessage]; + let iopubHit = 0; + future.onIOPub = msg => { + const targetMsg = iopubMessages[iopubHit]; + expect(msg.header.msg_id).to.equal(targetMsg.header.msg_id); + iopubHit = iopubHit + 1; + }; + + // Check our reply messages + const replyMessages = [replyMessage]; + let replyHit = 0; + future.onReply = msg => { + const targetMsg = replyMessages[replyHit]; + expect(msg.header.msg_id).to.equal(targetMsg.header.msg_id); + replyHit = replyHit + 1; + }; + + // Check our status changes + const statusChanges = ['busy', 'idle']; + let statusHit = 0; + const statusHandler: Slot = (_sender: RawKernel, args: Kernel.Status) => { + const targetStatus = statusChanges[statusHit]; + expect(rawKernel.status).to.equal(targetStatus); + expect(args).to.equal(targetStatus); + statusHit = statusHit + 1; + }; + rawKernel.statusChanged.connect(statusHandler); + + await future.done; + expect(iopubHit).to.equal(iopubMessages.length); + expect(replyHit).to.equal(replyMessages.length); + expect(statusHit).to.equal(statusChanges.length); + }); }); }); From 752063d7e04e6f6f1f98b723c72c921298daa88d Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Mon, 23 Mar 2020 11:02:35 -0700 Subject: [PATCH 005/725] More tests for enchannel rewrite (#10716) * Preliminary idea building * Get test to actually start a kernel * Socket event emitter * Send working * Fix result messages * Add another comment to make sure to fix the potential stack issue * Fix hygiene and some code review comments. * resolve promise stack * Some more test infrastructure * Add more tests * More tests * Fix up after merge --- .vscode/launch.json | 2 +- .../raw-kernel/enchannelJMPConnection.ts | 1 + .../raw-kernel/rawKernel.functional.test.ts | 215 ++++++++++++++---- 3 files changed, 178 insertions(+), 40 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 65ac14a02553..91edb211b1d8 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -234,7 +234,7 @@ "--ui=tdd", "--recursive", "--colors", - //"--grep", "", + //"--grep", "", "--timeout=300000" ], "env": { diff --git a/src/client/datascience/raw-kernel/enchannelJMPConnection.ts b/src/client/datascience/raw-kernel/enchannelJMPConnection.ts index 60bda1808b1c..b703e6dcb0ef 100644 --- a/src/client/datascience/raw-kernel/enchannelJMPConnection.ts +++ b/src/client/datascience/raw-kernel/enchannelJMPConnection.ts @@ -34,6 +34,7 @@ export class EnchannelJMPConnection implements IJMPConnection { public dispose(): void { if (this.mainChannel) { this.mainChannel.unsubscribe(); + this.mainChannel = undefined; } } } diff --git a/src/test/datascience/raw-kernel/rawKernel.functional.test.ts b/src/test/datascience/raw-kernel/rawKernel.functional.test.ts index 896d7824fa8f..ff6a8060d3ba 100644 --- a/src/test/datascience/raw-kernel/rawKernel.functional.test.ts +++ b/src/test/datascience/raw-kernel/rawKernel.functional.test.ts @@ -7,6 +7,7 @@ import * as fs from 'fs-extra'; import { noop } from 'jquery'; import * as os from 'os'; import * as path from 'path'; +import { Observable } from 'rxjs'; import * as uuid from 'uuid/v4'; import { IPythonExecutionFactory, ObservableExecutionResult } from '../../../client/common/process/types'; import { createDeferred } from '../../../client/common/utils/async'; @@ -19,6 +20,8 @@ suite('DataScience raw kernel tests', () => { let enchannelConnection: IJMPConnection; let connectionFile: string; let kernelProcResult: ObservableExecutionResult; + let messageObservable: Observable>; + let sessionId: string; const connectionInfo = { shell_port: 57718, iopub_port: 57719, @@ -40,47 +43,66 @@ suite('DataScience raw kernel tests', () => { // tslint:disable-next-line: no-invalid-this this.skip(); } else { - enchannelConnection = ioc.get(IJMPConnection); - - // Find our jupyter interpreter - const interpreter = await ioc.getJupyterCapableInterpreter(); - assert.ok(interpreter, 'No jupyter interpreter found'); - // Start our kernel - const execFactory = ioc.get(IPythonExecutionFactory); - const env = await execFactory.createActivatedEnvironment({ interpreter }); - - connectionFile = path.join(os.tmpdir(), `tmp_${Date.now()}_k.json`); - await fs.writeFile(connectionFile, JSON.stringify(connectionInfo), { encoding: 'utf-8', flag: 'w' }); - - // Keep kernel alive while the tests are running. - kernelProcResult = env.execObservable(['-m', 'ipykernel_launcher', '-f', connectionFile], { - throwOnStdErr: false - }); - kernelProcResult.out.subscribe( - out => { - console.log(out.out); - }, - error => { - console.error(error); - }, - () => { - enchannelConnection.dispose(); - } - ); + await connectToKernel(57718); } }); teardown(async () => { + await disconnectFromKernel(); + await ioc.dispose(); + }); + + async function connectToKernel(startPort: number) { + connectionInfo.stdin_port = startPort; + connectionInfo.shell_port = startPort + 1; + connectionInfo.iopub_port = startPort + 2; + connectionInfo.hb_port = startPort + 3; + connectionInfo.control_port = startPort + 4; + enchannelConnection = ioc.get(IJMPConnection); + + // Find our jupyter interpreter + const interpreter = await ioc.getJupyterCapableInterpreter(); + assert.ok(interpreter, 'No jupyter interpreter found'); + // Start our kernel + const execFactory = ioc.get(IPythonExecutionFactory); + const env = await execFactory.createActivatedEnvironment({ interpreter }); + + connectionFile = path.join(os.tmpdir(), `tmp_${Date.now()}_k.json`); + await fs.writeFile(connectionFile, JSON.stringify(connectionInfo), { encoding: 'utf-8', flag: 'w' }); + + // Keep kernel alive while the tests are running. + kernelProcResult = env.execObservable(['-m', 'ipykernel_launcher', '-f', connectionFile], { + throwOnStdErr: false + }); + kernelProcResult.out.subscribe( + out => { + console.log(out.out); + }, + error => { + console.error(error); + }, + () => { + enchannelConnection.dispose(); + } + ); + sessionId = uuid(); + await enchannelConnection.connect(connectionInfo); + messageObservable = new Observable(subscriber => { + enchannelConnection.subscribe(subscriber.next.bind(subscriber)); + }); + } + + async function disconnectFromKernel() { kernelProcResult?.proc?.kill(); try { await fs.remove(connectionFile); } catch { noop(); } - await ioc.dispose(); - }); + enchannelConnection.dispose(); + } - function createShutdownMessage(sessionId: string): KernelMessage.IMessage<'shutdown_request'> { + function createShutdownMessage(): KernelMessage.IMessage<'shutdown_request'> { return { channel: 'control', content: { @@ -99,16 +121,131 @@ suite('DataScience raw kernel tests', () => { }; } - // tslint:disable-next-line: no-function-expression - test('Basic iopub', async function() { - const reply = createDeferred(); - await enchannelConnection.connect(connectionInfo); - enchannelConnection.subscribe(msg => { - if (msg.header.msg_type === 'status') { - reply.resolve(); + function createExecutionMessage(code: string): KernelMessage.IExecuteRequestMsg { + return { + channel: 'shell', + content: { + code, + silent: false, + store_history: false + }, + header: { + date: Date.now().toString(), + msg_id: uuid(), + msg_type: 'execute_request', + session: sessionId, + username: 'user', + version: '5.1' + }, + parent_header: {}, + metadata: {} + }; + } + + function createInspectMessage(code: string): KernelMessage.IInspectRequestMsg { + return { + channel: 'shell', + content: { + code, + cursor_pos: code.length, + detail_level: 1 + }, + header: { + date: Date.now().toString(), + msg_id: uuid(), + msg_type: 'inspect_request', + session: sessionId, + username: 'user', + version: '5.1' + }, + parent_header: {}, + metadata: {} + }; + } + + function sendMessage( + message: KernelMessage.IMessage + ): Promise[]> { + const waiter = createDeferred[]>(); + const replies: KernelMessage.IMessage[] = []; + let expectedReplyType = 'status'; + switch (message.header.msg_type) { + case 'shutdown_request': + expectedReplyType = 'shutdown_reply'; + break; + + case 'execute_request': + expectedReplyType = 'execute_reply'; + break; + + case 'inspect_request': + expectedReplyType = 'inspect_reply'; + break; + default: + break; + } + let foundReply = false; + let foundIdle = false; + const subscr = messageObservable.subscribe(m => { + replies.push(m); + if (m.header.msg_type === 'status') { + foundIdle = (m.content as any).execution_state === 'idle'; + } else if (m.header.msg_type === expectedReplyType) { + foundReply = true; + } + + if (m.header.msg_type === 'shutdown_reply') { + // Special case, status may never come after this. + waiter.resolve(replies); + } + if (!waiter.resolved && foundReply && foundIdle) { + waiter.resolve(replies); } }); - enchannelConnection.sendMessage(createShutdownMessage(uuid())); - await reply.promise; + enchannelConnection.sendMessage(message); + return waiter.promise.then(m => { + subscr.unsubscribe(); + return m; + }); + } + + test('Basic connection', async () => { + const replies = await sendMessage(createShutdownMessage()); + assert.ok( + replies.find(r => r.header.msg_type === 'shutdown_reply'), + 'Reply not sent for shutdown' + ); + }); + + test('Basic request', async () => { + const replies = await sendMessage(createExecutionMessage('a=1\na')); + const executeResult = replies.find(r => r.header.msg_type === 'execute_result'); + assert.ok(executeResult, 'Result not found'); + assert.equal((executeResult?.content as any).data['text/plain'], '1', 'Results were not computed'); + }); + + test('Multiple requests', async () => { + let replies = await sendMessage(createExecutionMessage('a=1\na')); + let executeResult = replies.find(r => r.header.msg_type === 'execute_result'); + assert.ok(executeResult, 'Result not found'); + replies = await sendMessage(createExecutionMessage('a=2\na')); + executeResult = replies.find(r => r.header.msg_type === 'execute_result'); + assert.ok(executeResult, 'Result 2 not found'); + assert.equal((executeResult?.content as any).data['text/plain'], '2', 'Results were not computed'); + replies = await sendMessage(createInspectMessage('a')); + const inspectResult = replies.find(r => r.header.msg_type === 'inspect_reply'); + assert.ok(inspectResult, 'Inspect result not found'); + assert.ok((inspectResult?.content as any).data['text/plain'], 'Inspect reply was not computed'); + }); + + test('Startup and shutdown', async () => { + let replies = await sendMessage(createExecutionMessage('a=1\na')); + let executeResult = replies.find(r => r.header.msg_type === 'execute_result'); + assert.ok(executeResult, 'Result not found'); + await disconnectFromKernel(); + await connectToKernel(57418); + replies = await sendMessage(createExecutionMessage('a=1\na')); + executeResult = replies.find(r => r.header.msg_type === 'execute_result'); + assert.ok(executeResult, 'Result not found'); }); }); From 6b9fc7f08b8d3c83e198c9def2f940565cfa874f Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Mon, 23 Mar 2020 15:49:35 -0700 Subject: [PATCH 006/725] requestInspect + requestComplete + tests for both (#10723) --- .../datascience/raw-kernel/rawFuture.ts | 5 +- .../datascience/raw-kernel/rawKernel.ts | 119 +++++++++++++----- src/test/datascience/raw-kernel/mockJMP.ts | 6 +- .../raw-kernel/rawKernel.unit.test.ts | 104 ++++++++++++--- 4 files changed, 182 insertions(+), 52 deletions(-) diff --git a/src/client/datascience/raw-kernel/rawFuture.ts b/src/client/datascience/raw-kernel/rawFuture.ts index 79b67156ddf1..babb0df0f1da 100644 --- a/src/client/datascience/raw-kernel/rawFuture.ts +++ b/src/client/datascience/raw-kernel/rawFuture.ts @@ -111,12 +111,15 @@ export class RawFuture< } private async handleIOPub(message: KernelMessage.IIOPubMessage): Promise { + // tslint:disable-next-line:no-require-imports + const jupyterLab = require('@jupyterlab/services') as typeof import('@jupyterlab/services'); + // RAWKERNEL: Check hooks process first? // tslint:disable-next-line:no-any await this.ioPub(message); // If we get an idle status message and a reply then we are done - if (KernelMessage.isStatusMsg(message) && message.content.execution_state === 'idle') { + if (jupyterLab.KernelMessage.isStatusMsg(message) && message.content.execution_state === 'idle') { this.idleSeen = true; if (this.replyMessage) { diff --git a/src/client/datascience/raw-kernel/rawKernel.ts b/src/client/datascience/raw-kernel/rawKernel.ts index 00720d8d641f..eb5551099a5b 100644 --- a/src/client/datascience/raw-kernel/rawKernel.ts +++ b/src/client/datascience/raw-kernel/rawKernel.ts @@ -101,6 +101,9 @@ export class RawKernel implements Kernel.IKernel { _metadata?: JSONObject ): Kernel.IShellFuture { if (this.jmpConnection) { + // tslint:disable-next-line:no-require-imports + const jupyterLab = require('@jupyterlab/services') as typeof import('@jupyterlab/services'); + // Build our execution message // Silent is supposed to be options, but in my testing the message was not passing // correctly without it, so specifying it here with default false @@ -111,29 +114,65 @@ export class RawKernel implements Kernel.IKernel { username: 'vscode', content: { ...content, silent: content.silent || false } }; - const executeMessage = KernelMessage.createMessage(executeOptions); + const executeMessage = jupyterLab.KernelMessage.createMessage( + executeOptions + ); - // Send off our message to our jmp connection - this.jmpConnection.sendMessage(executeMessage); + const newFuture = this.sendShellMessage(executeMessage, disposeOnDone || true); - // Create a future to watch for reply messages - const newFuture = new RawFuture( - executeMessage, - disposeOnDone || true - ); - this.futures.set( - newFuture.msg.header.msg_id, - newFuture as RawFuture + return newFuture as Kernel.IShellFuture; + } + + // RAWKERNEL: What should we do here? Throw? + // Probably should not get here if session is not available + throw new Error('No session available?'); + } + + public requestComplete( + content: KernelMessage.ICompleteRequestMsg['content'] + ): Promise { + if (this.jmpConnection) { + // tslint:disable-next-line:no-require-imports + const jupyterLab = require('@jupyterlab/services') as typeof import('@jupyterlab/services'); + + const completeOptions: KernelMessage.IOptions = { + session: this._clientId, + channel: 'shell', + msgType: 'complete_request', + username: 'vscode', + content + }; + const completeMessage = jupyterLab.KernelMessage.createMessage( + completeOptions ); - // Set our future to remove itself when disposed - const oldDispose = newFuture.dispose.bind(newFuture); - newFuture.dispose = () => { - this.futures.delete(newFuture.msg.header.msg_id); - return oldDispose(); + return this.sendShellMessage(completeMessage).done as Promise; + } + + // RAWKERNEL: What should we do here? Throw? + // Probably should not get here if session is not available + throw new Error('No session available?'); + } + + public requestInspect( + content: KernelMessage.IInspectRequestMsg['content'] + ): Promise { + if (this.jmpConnection) { + // tslint:disable-next-line:no-require-imports + const jupyterLab = require('@jupyterlab/services') as typeof import('@jupyterlab/services'); + + const inspectOptions: KernelMessage.IOptions = { + session: this._clientId, + channel: 'shell', + msgType: 'inspect_request', + username: 'vscode', + content }; + const inspectMessage = jupyterLab.KernelMessage.createMessage( + inspectOptions + ); - return newFuture; + return this.sendShellMessage(inspectMessage).done as Promise; } // RAWKERNEL: What should we do here? Throw? @@ -141,6 +180,35 @@ export class RawKernel implements Kernel.IKernel { throw new Error('No session available?'); } + public sendShellMessage( + message: KernelMessage.IShellMessage, + _expectReply?: boolean, + disposeOnDone?: boolean + ): Kernel.IShellFuture> { + if (this.jmpConnection) { + // First send our message + this.jmpConnection.sendMessage(message); + + // Next we need to build our future + const future = new RawFuture(message, disposeOnDone || true); + + // RAWKERNEL: DisplayID calculations need to happen here + this.futures.set(message.header.msg_id, future); + + // Set our future to remove itself when disposed + const oldDispose = future.dispose.bind(future); + future.dispose = () => { + this.futures.delete(future.msg.header.msg_id); + return oldDispose(); + }; + + return future as Kernel.IShellFuture>; + } + + // RAWKERNEL: sending without a connection + throw new Error('Attemping to send shell message without connection'); + } + // On dispose close down our connection and get rid of saved futures public dispose(): void { if (!this.isDisposed) { @@ -163,13 +231,6 @@ export class RawKernel implements Kernel.IKernel { public getSpec(): Promise { throw new Error('Not yet implemented'); } - public sendShellMessage( - _msg: KernelMessage.IShellMessage, - _expectReply?: boolean, - _disposeOnDone?: boolean - ): Kernel.IShellFuture> { - throw new Error('Not yet implemented'); - } public sendControlMessage( _msg: KernelMessage.IControlMessage, _expectReply?: boolean, @@ -189,16 +250,6 @@ export class RawKernel implements Kernel.IKernel { public requestKernelInfo(): Promise { throw new Error('Not yet implemented'); } - public requestComplete( - _content: KernelMessage.ICompleteRequestMsg['content'] - ): Promise { - throw new Error('Not yet implemented'); - } - public requestInspect( - _content: KernelMessage.IInspectRequestMsg['content'] - ): Promise { - throw new Error('Not yet implemented'); - } public requestHistory( _content: KernelMessage.IHistoryRequestMsg['content'] ): Promise { diff --git a/src/test/datascience/raw-kernel/mockJMP.ts b/src/test/datascience/raw-kernel/mockJMP.ts index 7f530cbea907..fbb376b9ad23 100644 --- a/src/test/datascience/raw-kernel/mockJMP.ts +++ b/src/test/datascience/raw-kernel/mockJMP.ts @@ -4,12 +4,16 @@ import { KernelMessage } from '@jupyterlab/services'; import { IJMPConnection, IJMPConnectionInfo } from '../../../client/datascience/types'; export class MockJMPConnection implements IJMPConnection { + public firstHeaderSeen: KernelMessage.IHeader | undefined; private callback: ((message: KernelMessage.IMessage) => void) | undefined; public async connect(_connectInfo: IJMPConnectionInfo): Promise { return; } - public sendMessage(_message: KernelMessage.IMessage): void { + public sendMessage(message: KernelMessage.IMessage): void { + if (!this.firstHeaderSeen) { + this.firstHeaderSeen = message.header; + } return; } public subscribe(handlerFunc: (message: KernelMessage.IMessage) => void): void { diff --git a/src/test/datascience/raw-kernel/rawKernel.unit.test.ts b/src/test/datascience/raw-kernel/rawKernel.unit.test.ts index 16670d744d69..615203bc59f5 100644 --- a/src/test/datascience/raw-kernel/rawKernel.unit.test.ts +++ b/src/test/datascience/raw-kernel/rawKernel.unit.test.ts @@ -120,14 +120,7 @@ suite('Data Science - RawKernel', () => { const future = rawKernel.requestExecute(executeContent, true, undefined); // First message is iopub busy status - const iopubBusyOptions: KernelMessage.IOptions = { - channel: 'iopub', - session: rawKernel.clientId, - msgType: 'status', - content: { execution_state: 'busy' } - }; - const iopubBusyMessage = KernelMessage.createMessage(iopubBusyOptions); - iopubBusyMessage.parent_header = future.msg.header; + const iopubBusyMessage = buildStatusMessage('busy', rawKernel.clientId, future.msg.header); // Post the message mockJmpConnection.messageBack(iopubBusyMessage); @@ -161,14 +154,7 @@ suite('Data Science - RawKernel', () => { mockJmpConnection.messageBack(iopubStreamMessage); // Finally an idle message - const iopubIdleOptions: KernelMessage.IOptions = { - channel: 'iopub', - session: rawKernel.clientId, - msgType: 'status', - content: { execution_state: 'idle' } - }; - const iopubIdleMessage = KernelMessage.createMessage(iopubIdleOptions); - iopubIdleMessage.parent_header = future.msg.header; + const iopubIdleMessage = buildStatusMessage('idle', rawKernel.clientId, future.msg.header); // Post the message mockJmpConnection.messageBack(iopubIdleMessage); @@ -221,5 +207,91 @@ suite('Data Science - RawKernel', () => { expect(replyHit).to.equal(replyMessages.length); expect(statusHit).to.equal(statusChanges.length); }); + + test('RawKernel requestInspect messages', async () => { + await rawKernel.connect(connectInfo); + + // Check our status at the start + expect(rawKernel.status).to.equal('unknown'); + + // Create future for inspect request + const inspectContent: KernelMessage.IInspectRequestMsg['content'] = { + code: 'testing', + cursor_pos: 0, + detail_level: 0 + }; + const inspectPromise = rawKernel.requestInspect(inspectContent); + + // Pull out our parent header + const parentHeader = mockJmpConnection.firstHeaderSeen as KernelMessage.IHeader<'inspect_request'>; + + // pump an idle message as we need idle and a reply to be done + const iopubIdleMessage = buildStatusMessage('idle', rawKernel.clientId, parentHeader); + + // Post the message + mockJmpConnection.messageBack(iopubIdleMessage); + + // Send a reply message into our connection + const replyOptions: KernelMessage.IOptions = { + channel: 'shell', + session: rawKernel.clientId, + msgType: 'inspect_reply', + parentHeader, + content: { status: 'ok', found: true, metadata: {}, data: { myData: 'myData' } } + }; + const replyMessage = KernelMessage.createMessage(replyOptions); + mockJmpConnection.messageBack(replyMessage); + + const reply = await inspectPromise; + expect(reply.header.msg_id).to.equal(replyMessage.header.msg_id); + }); + + test('RawKernel requestComplete messages', async () => { + await rawKernel.connect(connectInfo); + + // Check our status at the start + expect(rawKernel.status).to.equal('unknown'); + + // Create future for inspect request + const completeContent: KernelMessage.ICompleteRequestMsg['content'] = { + code: 'testing', + cursor_pos: 0 + }; + const inspectPromise = rawKernel.requestComplete(completeContent); + + // Pull out our parent header + const parentHeader = mockJmpConnection.firstHeaderSeen as KernelMessage.IHeader<'complete_request'>; + + // pump an idle message as we need idle and a reply to be done + const iopubIdleMessage = buildStatusMessage('idle', rawKernel.clientId, parentHeader); + + // Post the message + mockJmpConnection.messageBack(iopubIdleMessage); + + // Send a reply message into our connection + const replyOptions: KernelMessage.IOptions = { + channel: 'shell', + session: rawKernel.clientId, + msgType: 'complete_reply', + parentHeader, + content: { status: 'ok', metadata: {}, cursor_start: 0, cursor_end: 0, matches: ['testing'] } + }; + const replyMessage = KernelMessage.createMessage(replyOptions); + mockJmpConnection.messageBack(replyMessage); + + const reply = await inspectPromise; + expect(reply.header.msg_id).to.equal(replyMessage.header.msg_id); + }); }); }); + +function buildStatusMessage(status: Kernel.Status, session: string, parentHeader: KernelMessage.IHeader) { + const iopubStatusOptions: KernelMessage.IOptions = { + channel: 'iopub', + session, + msgType: 'status', + parentHeader, + content: { execution_state: status } + }; + return KernelMessage.createMessage(iopubStatusOptions); +} From 927f75d3319033f231355b007941432540315121 Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Tue, 24 Mar 2020 04:36:24 -0700 Subject: [PATCH 007/725] send inputreply expectreply and tests (#10737) --- .../datascience/raw-kernel/rawFuture.ts | 15 +++- .../datascience/raw-kernel/rawKernel.ts | 32 ++++++-- src/test/datascience/raw-kernel/mockJMP.ts | 4 + .../raw-kernel/rawFuture.unit.test.ts | 73 ++++++++++++++++++- .../raw-kernel/rawKernel.unit.test.ts | 35 ++++++++- 5 files changed, 143 insertions(+), 16 deletions(-) diff --git a/src/client/datascience/raw-kernel/rawFuture.ts b/src/client/datascience/raw-kernel/rawFuture.ts index babb0df0f1da..0011f671616f 100644 --- a/src/client/datascience/raw-kernel/rawFuture.ts +++ b/src/client/datascience/raw-kernel/rawFuture.ts @@ -21,13 +21,19 @@ export class RawFuture< private reply: (msg: REPLY) => void | PromiseLike = noop; private replyMessage: REPLY | undefined; private disposeOnDone: boolean; - private idleSeen: boolean; + private idleSeen: boolean = false; + private replySeen: boolean = false; - constructor(msg: REQUEST, disposeOnDone: boolean) { + constructor(msg: REQUEST, expectReply: boolean, disposeOnDone: boolean) { this.msg = msg; this.donePromise = createDeferred(); this.disposeOnDone = disposeOnDone; - this.idleSeen = false; + + // If we don't expect a reply then indicate that we've already seen one + // for done checks + if (!expectReply) { + this.replySeen = true; + } } get done(): Promise { @@ -122,7 +128,7 @@ export class RawFuture< if (jupyterLab.KernelMessage.isStatusMsg(message) && message.content.execution_state === 'idle') { this.idleSeen = true; - if (this.replyMessage) { + if (this.replySeen) { this.handleDone(); } } @@ -141,6 +147,7 @@ export class RawFuture< await this.reply(message); this.replyMessage = message; + this.replySeen = true; // If we've gotten an idle status message we are done now if (this.idleSeen) { diff --git a/src/client/datascience/raw-kernel/rawKernel.ts b/src/client/datascience/raw-kernel/rawKernel.ts index eb5551099a5b..62c8b627db74 100644 --- a/src/client/datascience/raw-kernel/rawKernel.ts +++ b/src/client/datascience/raw-kernel/rawKernel.ts @@ -118,7 +118,7 @@ export class RawKernel implements Kernel.IKernel { executeOptions ); - const newFuture = this.sendShellMessage(executeMessage, disposeOnDone || true); + const newFuture = this.sendShellMessage(executeMessage, true, disposeOnDone || true); return newFuture as Kernel.IShellFuture; } @@ -146,7 +146,7 @@ export class RawKernel implements Kernel.IKernel { completeOptions ); - return this.sendShellMessage(completeMessage).done as Promise; + return this.sendShellMessage(completeMessage, true).done as Promise; } // RAWKERNEL: What should we do here? Throw? @@ -172,7 +172,7 @@ export class RawKernel implements Kernel.IKernel { inspectOptions ); - return this.sendShellMessage(inspectMessage).done as Promise; + return this.sendShellMessage(inspectMessage, true).done as Promise; } // RAWKERNEL: What should we do here? Throw? @@ -182,7 +182,7 @@ export class RawKernel implements Kernel.IKernel { public sendShellMessage( message: KernelMessage.IShellMessage, - _expectReply?: boolean, + expectReply?: boolean, disposeOnDone?: boolean ): Kernel.IShellFuture> { if (this.jmpConnection) { @@ -190,7 +190,7 @@ export class RawKernel implements Kernel.IKernel { this.jmpConnection.sendMessage(message); // Next we need to build our future - const future = new RawFuture(message, disposeOnDone || true); + const future = new RawFuture(message, expectReply || false, disposeOnDone || true); // RAWKERNEL: DisplayID calculations need to happen here this.futures.set(message.header.msg_id, future); @@ -209,6 +209,25 @@ export class RawKernel implements Kernel.IKernel { throw new Error('Attemping to send shell message without connection'); } + public sendInputReply(content: KernelMessage.IInputReplyMsg['content']): void { + if (this.jmpConnection) { + // tslint:disable-next-line:no-require-imports + const jupyterLab = require('@jupyterlab/services') as typeof import('@jupyterlab/services'); + const inputOptions: KernelMessage.IOptions = { + session: this.clientId, + channel: 'stdin', + msgType: 'input_reply', + content + }; + const inputReplyMessage = jupyterLab.KernelMessage.createMessage( + inputOptions + ); + + // Send off our input reply no futures or promises + this.jmpConnection.sendMessage(inputReplyMessage); + } + } + // On dispose close down our connection and get rid of saved futures public dispose(): void { if (!this.isDisposed) { @@ -271,9 +290,6 @@ export class RawKernel implements Kernel.IKernel { ): Promise { throw new Error('Not yet implemented'); } - public sendInputReply(_content: KernelMessage.IInputReplyMsg['content']): void { - throw new Error('Not yet implemented'); - } public connectToComm(_targetName: string, _commId?: string): Kernel.IComm { throw new Error('Not yet implemented'); } diff --git a/src/test/datascience/raw-kernel/mockJMP.ts b/src/test/datascience/raw-kernel/mockJMP.ts index fbb376b9ad23..949879c85375 100644 --- a/src/test/datascience/raw-kernel/mockJMP.ts +++ b/src/test/datascience/raw-kernel/mockJMP.ts @@ -5,6 +5,7 @@ import { IJMPConnection, IJMPConnectionInfo } from '../../../client/datascience/ export class MockJMPConnection implements IJMPConnection { public firstHeaderSeen: KernelMessage.IHeader | undefined; + public messagesSeen: KernelMessage.IMessage[] = []; private callback: ((message: KernelMessage.IMessage) => void) | undefined; public async connect(_connectInfo: IJMPConnectionInfo): Promise { @@ -14,6 +15,9 @@ export class MockJMPConnection implements IJMPConnection { if (!this.firstHeaderSeen) { this.firstHeaderSeen = message.header; } + + this.messagesSeen.push(message); + return; } public subscribe(handlerFunc: (message: KernelMessage.IMessage) => void): void { diff --git a/src/test/datascience/raw-kernel/rawFuture.unit.test.ts b/src/test/datascience/raw-kernel/rawFuture.unit.test.ts index bc1c2532d199..dec3cd3d8033 100644 --- a/src/test/datascience/raw-kernel/rawFuture.unit.test.ts +++ b/src/test/datascience/raw-kernel/rawFuture.unit.test.ts @@ -3,7 +3,9 @@ import { KernelMessage } from '@jupyterlab/services'; import { expect } from 'chai'; import * as uuid from 'uuid/v4'; +import { noop } from '../../../client/common/utils/misc'; import { RawFuture } from '../../../client/datascience/raw-kernel/rawFuture'; +import { buildExecuteReplyMessage, buildStatusMessage } from './rawKernel.unit.test'; // tslint:disable: max-func-body-length suite('Data Science - RawFuture', () => { @@ -22,10 +24,75 @@ suite('Data Science - RawFuture', () => { content: { code: 'print("hello world")' } }; executeMessage = KernelMessage.createMessage(executeOptions); - rawFuture = new RawFuture(executeMessage, true); + rawFuture = new RawFuture(executeMessage, true, true); }); - test('Check our reply message channel', async () => { + test('RawFuture dispose', async () => { + // Set up some handlers + rawFuture.onReply = _msg => { + noop(); + }; + rawFuture.onIOPub = _msg => { + noop(); + }; + rawFuture.onStdin = _msg => { + noop(); + }; + + rawFuture.done.catch(reason => { + const error = reason as Error; + expect(error.message).to.equal('Disposed Future'); + }); + + // dispose of the future + rawFuture.dispose(); + + expect(rawFuture.onReply).to.equal(noop); + expect(rawFuture.onIOPub).to.equal(noop); + expect(rawFuture.onStdin).to.equal(noop); + expect(rawFuture.isDisposed).to.equal(true, 'Done promise not rejected ;on dispose'); + }); + + test('RawFuture Check future expect reply off', async () => { + // Since expect reply is turned off, the done should be resolved without a reply message + rawFuture = new RawFuture(executeMessage, false, true); + + const idleMessage = buildStatusMessage('idle', sessionID, executeMessage.header); + + await rawFuture.handleMessage(idleMessage); + + await rawFuture.done; + }); + + test('RawFuture Check future expect reply on, dispose on done on', async () => { + // Since expect reply is turned on, the done should be resolved with a reply and an idle status + const idleMessage = buildStatusMessage('idle', sessionID, executeMessage.header); + const replyMessage = buildExecuteReplyMessage(sessionID, executeMessage.header); + + await rawFuture.handleMessage(idleMessage); + await rawFuture.handleMessage(replyMessage); + + await rawFuture.done; + expect(rawFuture.isDisposed).to.equal(true, 'Future not disposed on done'); + }); + + test('RawFuture Check future dispose on done off', async () => { + // Turn off dispose on done + rawFuture = new RawFuture(executeMessage, true, false); + + const idleMessage = buildStatusMessage('idle', sessionID, executeMessage.header); + const replyMessage = buildExecuteReplyMessage(sessionID, executeMessage.header); + + await rawFuture.handleMessage(idleMessage); + await rawFuture.handleMessage(replyMessage); + + await rawFuture.done; + expect(rawFuture.isDisposed).to.equal(false, 'Future disposed when dispose on done turned off'); + rawFuture.dispose(); + expect(rawFuture.isDisposed).to.equal(true, 'Future not disposed when dispose called'); + }); + + test('RawFuture Check our reply message channel', async () => { const replyOptions: KernelMessage.IOptions = { channel: 'shell', session: sessionID, @@ -50,7 +117,7 @@ suite('Data Science - RawFuture', () => { await rawFuture.handleMessage(replyMessage); }); - test('Check our IOPub message channel', async () => { + test('RawFuture Check our IOPub message channel', async () => { const ioPubMessageOptions: KernelMessage.IOptions = { session: sessionID, msgType: 'stream', diff --git a/src/test/datascience/raw-kernel/rawKernel.unit.test.ts b/src/test/datascience/raw-kernel/rawKernel.unit.test.ts index 615203bc59f5..2535323a204d 100644 --- a/src/test/datascience/raw-kernel/rawKernel.unit.test.ts +++ b/src/test/datascience/raw-kernel/rawKernel.unit.test.ts @@ -282,10 +282,32 @@ suite('Data Science - RawKernel', () => { const reply = await inspectPromise; expect(reply.header.msg_id).to.equal(replyMessage.header.msg_id); }); + + test('RawKernel sendInput messages', async () => { + await rawKernel.connect(connectInfo); + + // Check our status at the start + expect(rawKernel.status).to.equal('unknown'); + + // Create future for inspect request + const inputReplyContent: KernelMessage.IInputReplyMsg['content'] = { + value: 'input', + status: 'ok' + }; + rawKernel.sendInputReply(inputReplyContent); + + expect(mockJmpConnection.messagesSeen.length).to.equal(1); + const messageIn = mockJmpConnection.messagesSeen[0] as KernelMessage.IInputReplyMsg; + expect(messageIn.header.msg_type).to.equal('input_reply'); + // Type system kept fussing on the value type, so just any it + // tslint:disable-next-line:no-any + expect((messageIn.content as any).value).to.equal('input'); + expect(messageIn.content.status).to.equal('ok'); + }); }); }); -function buildStatusMessage(status: Kernel.Status, session: string, parentHeader: KernelMessage.IHeader) { +export function buildStatusMessage(status: Kernel.Status, session: string, parentHeader: KernelMessage.IHeader) { const iopubStatusOptions: KernelMessage.IOptions = { channel: 'iopub', session, @@ -295,3 +317,14 @@ function buildStatusMessage(status: Kernel.Status, session: string, parentHeader }; return KernelMessage.createMessage(iopubStatusOptions); } + +export function buildExecuteReplyMessage(session: string, parentHeader: KernelMessage.IHeader<'execute_request'>) { + const replyOptions: KernelMessage.IOptions = { + channel: 'shell', + session: session, + msgType: 'execute_reply', + parentHeader, + content: { status: 'ok', execution_count: 1, payload: [], user_expressions: {} } + }; + return KernelMessage.createMessage(replyOptions); +} From 9158a2a11453da496114187c29599ab563735023 Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Tue, 24 Mar 2020 21:26:34 -0700 Subject: [PATCH 008/725] RawSession added, BaseJupyterSession class with raw and server versions (#10764) --- src/client/datascience/baseJupyterSession.ts | 178 ++++++++++++++++++ .../datascience/jupyter/jupyterExecution.ts | 2 +- .../datascience/jupyter/jupyterSession.ts | 148 +-------------- .../jupyter/kernels/kernelSwitcher.ts | 2 +- .../raw-kernel/rawJupyterSession.ts | 61 ++++++ .../datascience/raw-kernel/rawKernel.ts | 11 +- .../datascience/raw-kernel/rawSession.ts | 131 +++++++++++++ .../kernels/kernelSwitcher.unit.test.ts | 2 +- .../raw-kernel/rawJupyterSession.unit.test.ts | 46 +++++ .../raw-kernel/rawKernel.unit.test.ts | 7 +- .../raw-kernel/rawSession.unit.test.ts | 133 +++++++++++++ 11 files changed, 571 insertions(+), 150 deletions(-) create mode 100644 src/client/datascience/baseJupyterSession.ts create mode 100644 src/client/datascience/raw-kernel/rawJupyterSession.ts create mode 100644 src/client/datascience/raw-kernel/rawSession.ts create mode 100644 src/test/datascience/raw-kernel/rawJupyterSession.unit.test.ts create mode 100644 src/test/datascience/raw-kernel/rawSession.unit.test.ts diff --git a/src/client/datascience/baseJupyterSession.ts b/src/client/datascience/baseJupyterSession.ts new file mode 100644 index 000000000000..7803196313b3 --- /dev/null +++ b/src/client/datascience/baseJupyterSession.ts @@ -0,0 +1,178 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +'use strict'; +import { Kernel, KernelMessage, Session } from '@jupyterlab/services'; +import { JSONObject } from '@phosphor/coreutils'; +import { Slot } from '@phosphor/signaling'; +import { Event, EventEmitter } from 'vscode'; +import { ServerStatus } from '../../datascience-ui/interactive-common/mainState'; +import { waitForPromise } from '../common/utils/async'; +import * as localize from '../common/utils/localize'; +import { sendTelemetryEvent } from '../telemetry'; +import { Telemetry } from './constants'; +import { JupyterKernelPromiseFailedError } from './jupyter/kernels/jupyterKernelPromiseFailedError'; +import { LiveKernelModel } from './jupyter/kernels/types'; +import { IJupyterKernelSpec, IJupyterSession } from './types'; + +export type ISession = Session.ISession & { + /** + * Whether this is a remote session that we attached to. + * + * @type {boolean} + */ + isRemoteSession?: boolean; +}; + +/** + * Exception raised when starting a Jupyter Session fails. + * + * @export + * @class JupyterSessionStartError + * @extends {Error} + */ +export class JupyterSessionStartError extends Error { + constructor(originalException: Error) { + super(originalException.message); + this.stack = originalException.stack; + sendTelemetryEvent(Telemetry.StartSessionFailedJupyter); + } +} + +/* +BaseJupyterSession represention functionality shared by a session regardless of if you are connected to a +server via JupyterLabs interfaces or via a raw ZMQ connection to a kernel. Raw classes implement the +jupyterlab interfaces so shared functionality that goes through the ISession or IKernel should go in here +*/ +export abstract class BaseJupyterSession implements IJupyterSession { + protected session: ISession | undefined; + protected connected: boolean = false; + protected onStatusChangedEvent: EventEmitter = new EventEmitter(); + protected statusHandler: Slot; + + constructor() { + this.statusHandler = this.onStatusChanged.bind(this); + } + + // Abstracts for each Session type to implement + public abstract async shutdown(): Promise; + public abstract async restart(timeout: number): Promise; + public abstract async changeKernel(kernel: IJupyterKernelSpec | LiveKernelModel, timeoutMS: number): Promise; + public abstract async waitForIdle(timeout: number): Promise; + + // When we are disposed, call our shutdown function + public dispose(): Promise { + return this.shutdown(); + } + + public async interrupt(timeout: number): Promise { + if (this.session && this.session.kernel) { + // Listen for session status changes + this.session.statusChanged.connect(this.statusHandler); + + await this.waitForKernelPromise( + this.session.kernel.interrupt(), + timeout, + localize.DataScience.interruptingKernelFailed() + ); + } + } + + // Return if we are connected to an active kernel + public get isConnected(): boolean { + return this.connected; + } + + // Return the current kernel status of our server + public get status(): ServerStatus { + return this.getServerStatus(); + } + + // Event for server status changes + public get onSessionStatusChanged(): Event { + if (!this.onStatusChangedEvent) { + this.onStatusChangedEvent = new EventEmitter(); + } + return this.onStatusChangedEvent.event; + } + + public requestExecute( + content: KernelMessage.IExecuteRequestMsg['content'], + disposeOnDone?: boolean, + metadata?: JSONObject + ): Kernel.IShellFuture | undefined { + return this.session && this.session.kernel + ? this.session.kernel.requestExecute(content, disposeOnDone, metadata) + : undefined; + } + + public requestInspect( + content: KernelMessage.IInspectRequestMsg['content'] + ): Promise { + return this.session && this.session.kernel + ? this.session.kernel.requestInspect(content) + : Promise.resolve(undefined); + } + + public requestComplete( + content: KernelMessage.ICompleteRequestMsg['content'] + ): Promise { + return this.session && this.session.kernel + ? this.session.kernel.requestComplete(content) + : Promise.resolve(undefined); + } + + public sendInputReply(content: string) { + if (this.session && this.session.kernel) { + // tslint:disable-next-line: no-any + this.session.kernel.sendInputReply({ value: content, status: 'ok' }); + } + } + + // Respond to status changes on the session + protected onStatusChanged(_s: Session.ISession) { + if (this.onStatusChangedEvent) { + this.onStatusChangedEvent.fire(this.getServerStatus()); + } + } + + private async waitForKernelPromise( + kernelPromise: Promise, + timeout: number, + errorMessage: string + ): Promise { + // Wait for this kernel promise to happen + try { + await waitForPromise(kernelPromise, timeout); + } catch (e) { + if (!e) { + // We timed out. Throw a specific exception + throw new JupyterKernelPromiseFailedError(errorMessage); + } + throw e; + } + } + + private getServerStatus(): ServerStatus { + if (this.session) { + switch (this.session.kernel.status) { + case 'busy': + return ServerStatus.Busy; + case 'dead': + return ServerStatus.Dead; + case 'idle': + case 'connected': + return ServerStatus.Idle; + case 'restarting': + case 'autorestarting': + case 'reconnecting': + return ServerStatus.Restarting; + case 'starting': + return ServerStatus.Starting; + default: + return ServerStatus.NotStarted; + } + } + + return ServerStatus.NotStarted; + } +} diff --git a/src/client/datascience/jupyter/jupyterExecution.ts b/src/client/datascience/jupyter/jupyterExecution.ts index 7053a06c11b8..aa33012ad445 100644 --- a/src/client/datascience/jupyter/jupyterExecution.ts +++ b/src/client/datascience/jupyter/jupyterExecution.ts @@ -14,6 +14,7 @@ import { StopWatch } from '../../common/utils/stopWatch'; import { IInterpreterService, PythonInterpreter } from '../../interpreter/contracts'; import { IServiceContainer } from '../../ioc/types'; import { captureTelemetry, sendTelemetryEvent } from '../../telemetry'; +import { JupyterSessionStartError } from '../baseJupyterSession'; import { Commands, Telemetry } from '../constants'; import { IConnection, @@ -25,7 +26,6 @@ import { INotebookServerOptions } from '../types'; import { JupyterSelfCertsError } from './jupyterSelfCertsError'; -import { JupyterSessionStartError } from './jupyterSession'; import { createRemoteConnectionInfo } from './jupyterUtils'; import { JupyterWaitForIdleError } from './jupyterWaitForIdleError'; import { JupyterZMQBinariesNotFoundError } from './jupyterZMQBinariesNotFoundError'; diff --git a/src/client/datascience/jupyter/jupyterSession.ts b/src/client/datascience/jupyter/jupyterSession.ts index be26ef274e02..b3252bedec11 100644 --- a/src/client/datascience/jupyter/jupyterSession.ts +++ b/src/client/datascience/jupyter/jupyterSession.ts @@ -13,9 +13,7 @@ import { import { JSONObject } from '@phosphor/coreutils'; import { Slot } from '@phosphor/signaling'; import * as uuid from 'uuid/v4'; -import { Event, EventEmitter } from 'vscode'; import { CancellationToken } from 'vscode-jsonrpc'; -import { ServerStatus } from '../../../datascience-ui/interactive-common/mainState'; import { Cancellation } from '../../common/cancellation'; import { isTestExecution } from '../../common/constants'; import { traceError, traceInfo, traceWarning } from '../../common/logger'; @@ -23,48 +21,21 @@ import { IOutputChannel } from '../../common/types'; import { sleep, waitForPromise } from '../../common/utils/async'; import * as localize from '../../common/utils/localize'; import { noop } from '../../common/utils/misc'; -import { captureTelemetry, sendTelemetryEvent } from '../../telemetry'; +import { captureTelemetry } from '../../telemetry'; +import { BaseJupyterSession, ISession, JupyterSessionStartError } from '../baseJupyterSession'; import { Telemetry } from '../constants'; import { reportAction } from '../progress/decorator'; import { ReportableAction } from '../progress/types'; -import { IConnection, IJupyterKernelSpec, IJupyterSession } from '../types'; +import { IConnection, IJupyterKernelSpec } from '../types'; import { JupyterInvalidKernelError } from './jupyterInvalidKernelError'; import { JupyterWaitForIdleError } from './jupyterWaitForIdleError'; -import { JupyterKernelPromiseFailedError } from './kernels/jupyterKernelPromiseFailedError'; import { KernelSelector } from './kernels/kernelSelector'; import { LiveKernelModel } from './kernels/types'; -type ISession = Session.ISession & { - /** - * Whether this is a remote session that we attached to. - * - * @type {boolean} - */ - isRemoteSession?: boolean; -}; - -/** - * Exception raised when starting a Jupyter Session fails. - * - * @export - * @class JupyterSessionStartError - * @extends {Error} - */ -export class JupyterSessionStartError extends Error { - constructor(originalException: Error) { - super(originalException.message); - this.stack = originalException.stack; - sendTelemetryEvent(Telemetry.StartSessionFailedJupyter); - } -} - -export class JupyterSession implements IJupyterSession { - private session: ISession | undefined; +export class JupyterSession extends BaseJupyterSession { private restartSessionPromise: Promise | undefined; private notebookFiles: Contents.IModel[] = []; - private onStatusChangedEvent: EventEmitter = new EventEmitter(); - private statusHandler: Slot; - private connected: boolean = false; + constructor( private connInfo: IConnection, private serverSettings: ServerConnection.ISettings, @@ -74,11 +45,7 @@ export class JupyterSession implements IJupyterSession { private readonly kernelSelector: KernelSelector, private readonly outputChannel: IOutputChannel ) { - this.statusHandler = this.onStatusChanged.bind(this); - } - - public dispose(): Promise { - return this.shutdown(); + super(); } public async shutdown(): Promise { @@ -115,17 +82,6 @@ export class JupyterSession implements IJupyterSession { traceInfo('Shutdown session -- complete'); } - public get onSessionStatusChanged(): Event { - if (!this.onStatusChangedEvent) { - this.onStatusChangedEvent = new EventEmitter(); - } - return this.onStatusChangedEvent.event; - } - - public get status(): ServerStatus { - return this.getServerStatus(); - } - @reportAction(ReportableAction.JupyterSessionWaitForIdleSession) public async waitForIdle(timeout: number): Promise { // Wait for idle on this session @@ -178,28 +134,12 @@ export class JupyterSession implements IJupyterSession { } } - public async interrupt(timeout: number): Promise { - if (this.session && this.session.kernel) { - // Listen for session status changes - this.session.statusChanged.connect(this.statusHandler); - - await this.waitForKernelPromise( - this.session.kernel.interrupt(), - timeout, - localize.DataScience.interruptingKernelFailed() - ); - } - } - public requestExecute( content: KernelMessage.IExecuteRequestMsg['content'], disposeOnDone?: boolean, metadata?: JSONObject ): Kernel.IShellFuture | undefined { - const result = - this.session && this.session.kernel - ? this.session.kernel.requestExecute(content, disposeOnDone, metadata) - : undefined; + const result = super.requestExecute(content, disposeOnDone, metadata); // It has been observed that starting the restart session slows down first time to execute a cell. // Solution is to start the restart session after the first execution of user code. if (!content.silent && result && !isTestExecution()) { @@ -208,29 +148,6 @@ export class JupyterSession implements IJupyterSession { return result; } - public requestInspect( - content: KernelMessage.IInspectRequestMsg['content'] - ): Promise { - return this.session && this.session.kernel - ? this.session.kernel.requestInspect(content) - : Promise.resolve(undefined); - } - - public requestComplete( - content: KernelMessage.ICompleteRequestMsg['content'] - ): Promise { - return this.session && this.session.kernel - ? this.session.kernel.requestComplete(content) - : Promise.resolve(undefined); - } - - public sendInputReply(content: string) { - if (this.session && this.session.kernel) { - // tslint:disable-next-line: no-any - this.session.kernel.sendInputReply({ value: content, status: 'ok' }); - } - } - public async connect(cancelToken?: CancellationToken): Promise { if (!this.connInfo) { throw new Error(localize.DataScience.sessionDisposed()); @@ -251,10 +168,6 @@ export class JupyterSession implements IJupyterSession { this.connected = true; } - public get isConnected(): boolean { - return this.connected; - } - public async changeKernel(kernel: IJupyterKernelSpec | LiveKernelModel, timeoutMS: number): Promise { let newSession: ISession | undefined; @@ -299,30 +212,6 @@ export class JupyterSession implements IJupyterSession { this.restartSessionPromise = this.createRestartSession(this.serverSettings, kernel, this.contentsManager); } - private getServerStatus(): ServerStatus { - if (this.session) { - switch (this.session.kernel.status) { - case 'busy': - return ServerStatus.Busy; - case 'dead': - return ServerStatus.Dead; - case 'idle': - case 'connected': - return ServerStatus.Idle; - case 'restarting': - case 'autorestarting': - case 'reconnecting': - return ServerStatus.Restarting; - case 'starting': - return ServerStatus.Starting; - default: - return ServerStatus.NotStarted; - } - } - - return ServerStatus.NotStarted; - } - private startRestartSession() { if (!this.restartSessionPromise && this.session && this.contentsManager) { this.restartSessionPromise = this.createRestartSession( @@ -457,29 +346,6 @@ export class JupyterSession implements IJupyterSession { } } - private async waitForKernelPromise( - kernelPromise: Promise, - timeout: number, - errorMessage: string - ): Promise { - // Wait for this kernel promise to happen - try { - return await waitForPromise(kernelPromise, timeout); - } catch (e) { - if (!e) { - // We timed out. Throw a specific exception - throw new JupyterKernelPromiseFailedError(errorMessage); - } - throw e; - } - } - - private onStatusChanged(_s: Session.ISession) { - if (this.onStatusChangedEvent) { - this.onStatusChangedEvent.fire(this.getServerStatus()); - } - } - private async shutdownSession( session: ISession | undefined, statusHandler: Slot | undefined diff --git a/src/client/datascience/jupyter/kernels/kernelSwitcher.ts b/src/client/datascience/jupyter/kernels/kernelSwitcher.ts index c1ba394d8080..a0ad3da3c040 100644 --- a/src/client/datascience/jupyter/kernels/kernelSwitcher.ts +++ b/src/client/datascience/jupyter/kernels/kernelSwitcher.ts @@ -9,10 +9,10 @@ import { IApplicationShell } from '../../../common/application/types'; import { IConfigurationService, Resource } from '../../../common/types'; import { Common, DataScience } from '../../../common/utils/localize'; import { StopWatch } from '../../../common/utils/stopWatch'; +import { JupyterSessionStartError } from '../../baseJupyterSession'; import { Commands, Settings } from '../../constants'; import { IConnection, IJupyterKernelSpec, IJupyterSessionManagerFactory, INotebook } from '../../types'; import { JupyterInvalidKernelError } from '../jupyterInvalidKernelError'; -import { JupyterSessionStartError } from '../jupyterSession'; import { KernelSelector, KernelSpecInterpreter } from './kernelSelector'; import { LiveKernelModel } from './types'; diff --git a/src/client/datascience/raw-kernel/rawJupyterSession.ts b/src/client/datascience/raw-kernel/rawJupyterSession.ts new file mode 100644 index 000000000000..570c8b8e5cc1 --- /dev/null +++ b/src/client/datascience/raw-kernel/rawJupyterSession.ts @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +'use strict'; +import { CancellationToken } from 'vscode-jsonrpc'; +import { traceInfo } from '../../common/logger'; +import { BaseJupyterSession } from '../baseJupyterSession'; +import { LiveKernelModel } from '../jupyter/kernels/types'; +import { reportAction } from '../progress/decorator'; +import { ReportableAction } from '../progress/types'; +import { RawSession } from '../raw-kernel/rawSession'; +import { IJMPConnection, IJMPConnectionInfo, IJupyterKernelSpec } from '../types'; + +/* +RawJupyterSession is the implementation of IJupyterSession that instead off +connecting to JupyterLab services it instead connects to a kernel directly +through ZMQ. +It's responsible for translating our IJupyterSession interface into the +jupyterlabs interface as well as starting up and connecting to a raw session +*/ +export class RawJupyterSession extends BaseJupyterSession { + private rawSession: RawSession; + + constructor(connection: IJMPConnection) { + super(); + this.rawSession = new RawSession(connection); + this.session = this.rawSession; + } + + public async shutdown(): Promise { + if (this.session) { + this.session.dispose(); + this.session = undefined; + } + + if (this.onStatusChangedEvent) { + this.onStatusChangedEvent.dispose(); + } + traceInfo('Shutdown session -- complete'); + } + + @reportAction(ReportableAction.JupyterSessionWaitForIdleSession) + public async waitForIdle(_timeout: number): Promise { + // RawKernels are good to go right away + } + + public async restart(_timeout: number): Promise { + throw new Error('Not implemented'); + } + + // RAWKERNEL: Cancel token routed down? + public async connect(connectionInfo: IJMPConnectionInfo, _cancelToken?: CancellationToken): Promise { + await this.rawSession.connect(connectionInfo); + + // At this point we are connected and ready to work + this.connected = true; + } + + public async changeKernel(_kernel: IJupyterKernelSpec | LiveKernelModel, _timeoutMS: number): Promise { + throw new Error('Not implemented'); + } +} diff --git a/src/client/datascience/raw-kernel/rawKernel.ts b/src/client/datascience/raw-kernel/rawKernel.ts index 62c8b627db74..65e66878599f 100644 --- a/src/client/datascience/raw-kernel/rawKernel.ts +++ b/src/client/datascience/raw-kernel/rawKernel.ts @@ -36,7 +36,7 @@ export class RawKernel implements Kernel.IKernel { // IKernelConnection properties get id(): string { - throw new Error('Not yet implemented'); + return this._id; } get name(): string { throw new Error('Not yet implemented'); @@ -70,6 +70,7 @@ export class RawKernel implements Kernel.IKernel { private jmpConnection: IJMPConnection; private messageChain: Promise = Promise.resolve(); + private _id: string; private _clientId: string; private _status: Kernel.Status; private _statusChanged: Signal; @@ -81,11 +82,13 @@ export class RawKernel implements Kernel.IKernel { >(); // JMP connection should be injected, but no need to yet until it actually exists - constructor(connection: IJMPConnection) { - this._clientId = uuid(); + constructor(jmpConnection: IJMPConnection, clientID: string) { + // clientID is controlled by the session as we keep the same id + this._clientId = clientID; + this._id = uuid(); this._status = 'unknown'; this._statusChanged = new Signal(this); - this.jmpConnection = connection; + this.jmpConnection = jmpConnection; } public async connect(connectInfo: IJMPConnectionInfo) { diff --git a/src/client/datascience/raw-kernel/rawSession.ts b/src/client/datascience/raw-kernel/rawSession.ts new file mode 100644 index 000000000000..dcfff0f1f09d --- /dev/null +++ b/src/client/datascience/raw-kernel/rawSession.ts @@ -0,0 +1,131 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +import { Kernel, KernelMessage, ServerConnection, Session } from '@jupyterlab/services'; +import { ISignal, Signal } from '@phosphor/signaling'; +import * as uuid from 'uuid/v4'; +import { IJMPConnection, IJMPConnectionInfo } from '../types'; +import { RawKernel } from './rawKernel'; + +/* +RawSession class implements a jupyterlab ISession object +This provides enough of the ISession interface so that our direct +ZMQ Kernel connection can pretend to be a jupyterlab Session +*/ +export class RawSession implements Session.ISession { + public isDisposed: boolean = false; + + // Note, ID is the ID of this session + // ClientID is the ID that we pass in messages to the kernel + // and is also the clientID of the active kernel + private _id: string; + private _clientID: string; + private _kernel: RawKernel; + private _statusChanged = new Signal(this); + + // RAWKERNEL: Still just pass connection for now, we'll have to + // inject this further up the chain + constructor(connection: IJMPConnection) { + // Unique ID for this session instance + this._id = uuid(); + + // ID for our client JMP connection + this._clientID = uuid(); + + // Connect our kernel + this._kernel = new RawKernel(connection, this._clientID); + } + + public async connect(connectionInfo: IJMPConnectionInfo) { + await this._kernel.connect(connectionInfo); + + // Connect for status changes + this._kernel.statusChanged.connect(this.onKernelStatus, this); + } + + public dispose() { + if (!this.isDisposed) { + this._kernel.dispose(); + } + + this.isDisposed = true; + } + + // Return the ID, this is session's ID, not clientID for messages + get id(): string { + return this._id; + } + + // Return the current kernel for this session + get kernel(): Kernel.IKernelConnection { + return this._kernel; + } + + // Provide status changes for the attached kernel + get statusChanged(): ISignal { + return this._statusChanged; + } + + // Shutdown our session and kernel + public shutdown(): Promise { + this.dispose(); + // Normally the server session has to shutdown here with an await on a rest call + // but we just have a local connection, so dispose and resolve + return Promise.resolve(); + } + + // Not Implemented ISession + get terminated(): ISignal { + throw new Error('Not yet implemented'); + } + get kernelChanged(): ISignal { + throw new Error('Not yet implemented'); + } + get propertyChanged(): ISignal { + throw new Error('Not yet implemented'); + } + get iopubMessage(): ISignal { + throw new Error('Not yet implemented'); + } + get unhandledMessage(): ISignal { + throw new Error('Not yet implemented'); + } + get anyMessage(): ISignal { + throw new Error('Not yet implemented'); + } + get path(): string { + throw new Error('Not yet implemented'); + } + get name(): string { + throw new Error('Not yet implemented'); + } + get type(): string { + throw new Error('Not yet implemented'); + } + get serverSettings(): ServerConnection.ISettings { + throw new Error('Not yet implemented'); + } + get model(): Session.IModel { + throw new Error('Not yet implemented'); + } + get status(): Kernel.Status { + throw new Error('Not yet implemented'); + } + public setPath(_path: string): Promise { + throw new Error('Not yet implemented'); + } + public setName(_name: string): Promise { + throw new Error('Not yet implemented'); + } + public setType(_type: string): Promise { + throw new Error('Not yet implemented'); + } + public changeKernel(_options: Partial): Promise { + throw new Error('Not yet implemented'); + } + + // Private + // Send out a message when our kernel changes state + private onKernelStatus(_sender: Kernel.IKernelConnection, state: Kernel.Status) { + this._statusChanged.emit(state); + } +} diff --git a/src/test/datascience/jupyter/kernels/kernelSwitcher.unit.test.ts b/src/test/datascience/jupyter/kernels/kernelSwitcher.unit.test.ts index 98aa37aab540..2573da317de8 100644 --- a/src/test/datascience/jupyter/kernels/kernelSwitcher.unit.test.ts +++ b/src/test/datascience/jupyter/kernels/kernelSwitcher.unit.test.ts @@ -13,10 +13,10 @@ import { ConfigurationService } from '../../../../client/common/configuration/se import { IConfigurationService, IPythonSettings } from '../../../../client/common/types'; import { Common, DataScience } from '../../../../client/common/utils/localize'; import { Architecture } from '../../../../client/common/utils/platform'; +import { JupyterSessionStartError } from '../../../../client/datascience/baseJupyterSession'; import { Commands } from '../../../../client/datascience/constants'; import { JupyterNotebookBase } from '../../../../client/datascience/jupyter/jupyterNotebook'; import { JupyterServerWrapper } from '../../../../client/datascience/jupyter/jupyterServerWrapper'; -import { JupyterSessionStartError } from '../../../../client/datascience/jupyter/jupyterSession'; import { JupyterSessionManagerFactory } from '../../../../client/datascience/jupyter/jupyterSessionManagerFactory'; import { KernelSelector } from '../../../../client/datascience/jupyter/kernels/kernelSelector'; import { KernelSwitcher } from '../../../../client/datascience/jupyter/kernels/kernelSwitcher'; diff --git a/src/test/datascience/raw-kernel/rawJupyterSession.unit.test.ts b/src/test/datascience/raw-kernel/rawJupyterSession.unit.test.ts new file mode 100644 index 000000000000..64a91155e72d --- /dev/null +++ b/src/test/datascience/raw-kernel/rawJupyterSession.unit.test.ts @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +import { assert } from 'chai'; +import * as sinon from 'sinon'; +import { mock } from 'ts-mockito'; +import { RawJupyterSession } from '../../../client/datascience/raw-kernel/rawJupyterSession'; +import { IJMPConnection, IJMPConnectionInfo } from '../../../client/datascience/types'; + +// Note: The jupyterSession.unit.test.ts tests cover much of the base class functionality +// and lower level is handled by RawFuture, RawKernel, and RawSession +// tslint:disable: max-func-body-length +suite('Data Science - RawJupyterSession', () => { + let rawJupyterSession: RawJupyterSession; + let jmpConnection: IJMPConnection; + let connectInfo: IJMPConnectionInfo; + + setup(() => { + jmpConnection = mock(); + connectInfo = { + version: 0, + transport: 'tcp', + ip: '127.0.0.1', + shell_port: 55196, + iopub_port: 55197, + stdin_port: 55198, + hb_port: 55200, + control_port: 55199, + signature_scheme: 'hmac-sha256', + key: 'adaf9032-487d222a85026db284c3d5e7' + }; + rawJupyterSession = new RawJupyterSession(jmpConnection); + }); + + test('RawJupyterSession - connect', async () => { + await rawJupyterSession.connect(connectInfo); + + assert.isTrue(rawJupyterSession.isConnected); + }); + + test('RawJupyterSession - shutdown on dispose', async () => { + const shutdown = sinon.stub(rawJupyterSession, 'shutdown'); + shutdown.resolves(); + await rawJupyterSession.dispose(); + assert.isTrue(shutdown.calledOnce); + }); +}); diff --git a/src/test/datascience/raw-kernel/rawKernel.unit.test.ts b/src/test/datascience/raw-kernel/rawKernel.unit.test.ts index 2535323a204d..dfb9fef06513 100644 --- a/src/test/datascience/raw-kernel/rawKernel.unit.test.ts +++ b/src/test/datascience/raw-kernel/rawKernel.unit.test.ts @@ -4,6 +4,7 @@ import { Kernel, KernelMessage } from '@jupyterlab/services'; import { Slot } from '@phosphor/signaling'; import { assert, expect } from 'chai'; import { anything, deepEqual, instance, mock, verify, when } from 'ts-mockito'; +import * as uuid from 'uuid/v4'; import { RawKernel } from '../../../client/datascience/raw-kernel/rawKernel'; import { IJMPConnection, IJMPConnectionInfo } from '../../../client/datascience/types'; import { MockJMPConnection } from './mockJMP'; @@ -19,7 +20,7 @@ suite('Data Science - RawKernel', () => { jmpConnection = mock(); when(jmpConnection.connect(anything())).thenResolve(); when(jmpConnection.subscribe(anything())).thenReturn(); - rawKernel = new RawKernel(instance(jmpConnection)); + rawKernel = new RawKernel(instance(jmpConnection), uuid()); connectInfo = { version: 0, @@ -39,6 +40,8 @@ suite('Data Science - RawKernel', () => { await rawKernel.connect(connectInfo); verify(jmpConnection.connect(deepEqual(connectInfo))).once(); verify(jmpConnection.subscribe(anything())).once(); + // Verify that we have a client id an a kernel id + expect(rawKernel.id).to.not.equal(rawKernel.clientId); }); test('RawKernel dispose should dispose the jmp', async () => { @@ -103,7 +106,7 @@ suite('Data Science - RawKernel', () => { setup(() => { mockJmpConnection = new MockJMPConnection(); - rawKernel = new RawKernel(mockJmpConnection); + rawKernel = new RawKernel(mockJmpConnection, uuid()); }); test('RawKernel executeRequest messages', async () => { diff --git a/src/test/datascience/raw-kernel/rawSession.unit.test.ts b/src/test/datascience/raw-kernel/rawSession.unit.test.ts new file mode 100644 index 000000000000..ae0893f98e0f --- /dev/null +++ b/src/test/datascience/raw-kernel/rawSession.unit.test.ts @@ -0,0 +1,133 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +import { Kernel, KernelMessage } from '@jupyterlab/services'; +import { Slot } from '@phosphor/signaling'; +import { expect } from 'chai'; +import { anything, deepEqual, instance, mock, verify, when } from 'ts-mockito'; +import { RawSession } from '../../../client/datascience/raw-kernel/rawSession'; +import { IJMPConnection, IJMPConnectionInfo } from '../../../client/datascience/types'; +import { MockJMPConnection } from './mockJMP'; +import { buildStatusMessage } from './rawKernel.unit.test'; + +// tslint:disable: max-func-body-length +suite('Data Science - RawSession', () => { + let rawSession: RawSession; + let connectInfo: IJMPConnectionInfo; + + suite('RawSession - basic JMP', () => { + let jmpConnection: IJMPConnection; + setup(() => { + jmpConnection = mock(); + when(jmpConnection.connect(anything())).thenResolve(); + when(jmpConnection.subscribe(anything())).thenReturn(); + rawSession = new RawSession(instance(jmpConnection)); + + connectInfo = { + version: 0, + transport: 'tcp', + ip: '127.0.0.1', + shell_port: 55196, + iopub_port: 55197, + stdin_port: 55198, + hb_port: 55200, + control_port: 55199, + signature_scheme: 'hmac-sha256', + key: 'adaf9032-487d222a85026db284c3d5e7' + }; + }); + + test('RawSession construct', async () => { + // Kernel status should be unknown + expect(rawSession.kernel.status).to.equal('unknown'); + + // The ID of the session is not the same as the kernel client id + expect(rawSession.kernel.clientId).to.not.equal(rawSession.id); + }); + + test('RawSession connect', async () => { + await rawSession.connect(connectInfo); + + // Did we hook up our connection + verify(jmpConnection.connect(deepEqual(connectInfo))).once(); + verify(jmpConnection.subscribe(anything())).once(); + // The ID of the session is not the same as the kernel client id + expect(rawSession.kernel.clientId).to.not.equal(rawSession.id); + expect(rawSession.kernel.id).to.not.equal(rawSession.id); + }); + + test('RawSession dispose', async () => { + // Kernel status should be unknown + expect(rawSession.kernel.status).to.equal('unknown'); + + // The ID of the session is not the same as the kernel client id + expect(rawSession.kernel.clientId).to.not.equal(rawSession.id); + expect(rawSession.kernel.id).to.not.equal(rawSession.id); + }); + }); + + suite('RawSession - mock JMP', () => { + let mockJmpConnection: MockJMPConnection; + setup(() => { + mockJmpConnection = new MockJMPConnection(); + rawSession = new RawSession(mockJmpConnection); + + connectInfo = { + version: 0, + transport: 'tcp', + ip: '127.0.0.1', + shell_port: 55196, + iopub_port: 55197, + stdin_port: 55198, + hb_port: 55200, + control_port: 55199, + signature_scheme: 'hmac-sha256', + key: 'adaf9032-487d222a85026db284c3d5e7' + }; + }); + + test('RawSession status updates', async () => { + await rawSession.connect(connectInfo); + + const statusChanges = ['busy', 'idle']; + let statusHit = 0; + const statusHandler: Slot = (_sender: RawSession, args: Kernel.Status) => { + const targetStatus = statusChanges[statusHit]; + expect(rawSession.kernel.status).to.equal(targetStatus); + expect(args).to.equal(targetStatus); + statusHit = statusHit + 1; + }; + rawSession.statusChanged.connect(statusHandler); + + // Create a future for an execute code request + const code = 'print("hello world")'; + const executeContent: KernelMessage.IExecuteRequestMsg['content'] = { + code + }; + const future = rawSession.kernel.requestExecute(executeContent, true, undefined); + + // 1. First message is iopub busy status + const iopubBusyMessage = buildStatusMessage('busy', rawSession.kernel.clientId, future.msg.header); + mockJmpConnection.messageBack(iopubBusyMessage); + + // 2. an idle message + const iopubIdleMessage = buildStatusMessage('idle', rawSession.kernel.clientId, future.msg.header); + mockJmpConnection.messageBack(iopubIdleMessage); + + // 3. Last thing back is a reply message + const replyOptions: KernelMessage.IOptions = { + channel: 'shell', + session: rawSession.kernel.clientId, + msgType: 'execute_reply', + content: { status: 'ok', execution_count: 1, payload: [], user_expressions: {} } + }; + const replyMessage = KernelMessage.createMessage(replyOptions); + replyMessage.parent_header = future.msg.header; + mockJmpConnection.messageBack(replyMessage); + + await future.done; + + // Did we hit the status changes that we expect + expect(statusHit).to.equal(statusChanges.length); + }); + }); +}); From 69860b3384b9e00a5f6f081b36ea2481551322ea Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Wed, 25 Mar 2020 16:03:14 -0700 Subject: [PATCH 009/725] add channel onto returned JupyterMessage from ZMQ (#10788) --- news/2 Fixes/10785.md | 1 + .../datascience/raw-kernel/enchannel-zmq-backend-6/index.ts | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 news/2 Fixes/10785.md diff --git a/news/2 Fixes/10785.md b/news/2 Fixes/10785.md new file mode 100644 index 000000000000..91d258f93eb1 --- /dev/null +++ b/news/2 Fixes/10785.md @@ -0,0 +1 @@ +Add channel property onto returned ZMQ messages. \ No newline at end of file diff --git a/src/client/datascience/raw-kernel/enchannel-zmq-backend-6/index.ts b/src/client/datascience/raw-kernel/enchannel-zmq-backend-6/index.ts index ba98b02a6774..b75b96f34548 100644 --- a/src/client/datascience/raw-kernel/enchannel-zmq-backend-6/index.ts +++ b/src/client/datascience/raw-kernel/enchannel-zmq-backend-6/index.ts @@ -237,11 +237,14 @@ export const createMainChannelFromSockets = ( return rxjs.fromEvent(socketEmitter, 'message').pipe( map( (body: any): JupyterMessage => { - return wireProtocol.decode( + const message = wireProtocol.decode( body, connectionInfo.key, connectionInfo.signature_scheme ) as any; + // Add on our channel property + message.channel = name; + return message; } ), publish(), From af72e88b8f42e3d8338ba3deb7936f63d81c824b Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Thu, 26 Mar 2020 14:11:50 -0700 Subject: [PATCH 010/725] Add display_id handling for messages (#10812) --- .../datascience/raw-kernel/rawKernel.ts | 154 +++++++++++++++++- .../raw-kernel/rawKernel.unit.test.ts | 101 ++++++++++-- 2 files changed, 241 insertions(+), 14 deletions(-) diff --git a/src/client/datascience/raw-kernel/rawKernel.ts b/src/client/datascience/raw-kernel/rawKernel.ts index 65e66878599f..cd940c1cacd9 100644 --- a/src/client/datascience/raw-kernel/rawKernel.ts +++ b/src/client/datascience/raw-kernel/rawKernel.ts @@ -3,6 +3,8 @@ import { Kernel, KernelMessage, ServerConnection } from '@jupyterlab/services'; import { JSONObject } from '@phosphor/coreutils'; import { ISignal, Signal } from '@phosphor/signaling'; +// tslint:disable-next-line: no-require-imports +import cloneDeep = require('lodash/cloneDeep'); import * as uuid from 'uuid/v4'; import { traceError } from '../../common/logger'; import { IJMPConnection, IJMPConnectionInfo } from '../types'; @@ -68,7 +70,13 @@ export class RawKernel implements Kernel.IKernel { public isDisposed: boolean = false; private jmpConnection: IJMPConnection; + // Message chain to handle our messages async, but in order private messageChain: Promise = Promise.resolve(); + // Mappings for display id tracking + private displayIdToParentIds = new Map(); + private msgIdToDisplayIds = new Map(); + // The current kernel session Id that we are working with + private kernelSession: String = ''; private _id: string; private _clientId: string; @@ -201,7 +209,7 @@ export class RawKernel implements Kernel.IKernel { // Set our future to remove itself when disposed const oldDispose = future.dispose.bind(future); future.dispose = () => { - this.futures.delete(future.msg.header.msg_id); + this.futureDisposed(future); return oldDispose(); }; @@ -321,8 +329,47 @@ export class RawKernel implements Kernel.IKernel { throw new Error('Not yet implemented'); } + // When a future is disposed this function is called to remove it from our + // various tracking lists + private futureDisposed(future: RawFuture) { + const messageId = future.msg.header.msg_id; + this.futures.delete(messageId); + + // Remove stored display id information. + const displayIds = this.msgIdToDisplayIds.get(messageId); + if (!displayIds) { + return; + } + + displayIds.forEach(displayId => { + const messageIds = this.displayIdToParentIds.get(displayId); + if (messageIds) { + const index = messageIds.indexOf(messageId); + if (index === -1) { + return; + } + + if (messageIds.length === 1) { + this.displayIdToParentIds.delete(displayId); + } else { + messageIds.splice(index, 1); + this.displayIdToParentIds.set(displayId, messageIds); + } + } + }); + + // Remove our message id from the mapping to display ids + this.msgIdToDisplayIds.delete(messageId); + } + // Message incoming from the JMP connection. Queue it up for processing private msgIn(message: KernelMessage.IMessage) { + // Always keep our kernel session id up to date with incoming messages + // on something like a restart this will update when the first message on the + // new session comes in we use this to check the validity of messages that we are + // currently handling + this.kernelSession = message.header.session; + // Add the message onto our message chain, we want to process them async // but in order so use a chain like this this.messageChain = this.messageChain @@ -336,18 +383,119 @@ export class RawKernel implements Kernel.IKernel { }); } + private async handleDisplayId(displayId: string, message: KernelMessage.IMessage): Promise { + // tslint:disable-next-line:no-require-imports + const jupyterLab = require('@jupyterlab/services') as typeof import('@jupyterlab/services'); + + const messageId = (message.parent_header as KernelMessage.IHeader).msg_id; + + // Get all parent ids for this display id + let parentIds = this.displayIdToParentIds.get(displayId); + + // If we have seen this id before + if (parentIds) { + // We need to create a new update display data message to update the parents + const updateMessage: KernelMessage.IMessage = { + header: cloneDeep(message.header), + parent_header: cloneDeep(message.parent_header), + metadata: cloneDeep(message.metadata), + content: cloneDeep(message.content), + channel: message.channel, + buffers: message.buffers ? message.buffers.slice() : [] + }; + updateMessage.header.msg_type = 'update_display_data'; + + // Now send it out to all the parents + await Promise.all( + parentIds.map(async parentId => { + const future = this.futures && this.futures.get(parentId); + if (future) { + await future.handleMessage(updateMessage); + } + }) + ); + } + + if (jupyterLab.KernelMessage.isUpdateDisplayDataMsg(message)) { + // End here for an update display data, indicate that we have handed it + // so it skip the normal displaying in handleMessage + return true; + } + + // For display_data message record the mapping from + // the displayId to the parent messageId + parentIds = this.displayIdToParentIds.get(displayId) ?? []; + if (parentIds.indexOf(messageId) === -1) { + parentIds.push(messageId); + } + this.displayIdToParentIds.set(displayId, parentIds); + + // Add to mapping of message -> display ids + const displayIds = this.msgIdToDisplayIds.get(messageId) ?? []; + if (displayIds.indexOf(messageId) === -1) { + displayIds.push(messageId); + } + this.msgIdToDisplayIds.set(messageId, displayIds); + + // Return false so message continues to get processed + return false; + } + + /* + Messages are handled async so there is a possibility that the kernel might be + disposed or restarted during handling. Throw an error here if our message that + we are handling is no longer valid. + */ + private checkMessageValid(message: KernelMessage.IMessage) { + if (this.isDisposed) { + throw new Error('Stop message handling on diposed kernel'); + } + + // kernelSession is updated when the first message from a new kernel session comes in + // in this case don't keep handling the old session messages + if (message.header.session !== this.kernelSession) { + throw new Error('Stop message handling on message from old session'); + } + } + // Handle a new message arriving from JMP connection private async handleMessage(message: KernelMessage.IMessage): Promise { - // RAWKERNEL: display_data messages can route based on their id here first + // IANHU: CONVERT TO USING ONE REQUIRE? + // tslint:disable-next-line:no-require-imports + const jupyterLab = require('@jupyterlab/services') as typeof import('@jupyterlab/services'); + + let handled = false; + + // Check to see if we have the right type of message for a display id + if ( + message.parent_header && + message.channel === 'iopub' && + (jupyterLab.KernelMessage.isDisplayDataMsg(message) || + jupyterLab.KernelMessage.isUpdateDisplayDataMsg(message) || + jupyterLab.KernelMessage.isExecuteResultMsg(message)) + ) { + // Display id can be found in transient message content + // https://jupyter-client.readthedocs.io/en/stable/messaging.html#display-data + const displayId = message.content.transient?.display_id; + if (displayId) { + handled = await this.handleDisplayId(displayId, message); + + // After await check the validity of our message + this.checkMessageValid(message); + } + } // Look up in our future list and see if a future needs to be updated on this message - if (message.parent_header) { + if (!handled && message.parent_header) { const parentHeader = message.parent_header as KernelMessage.IHeader; const parentFuture = this.futures.get(parentHeader.msg_id); if (parentFuture) { // Let the parent future message handle it here await parentFuture.handleMessage(message); + + // After await check the validity of our message + this.checkMessageValid(message); } else { if (message.header.session === this._clientId && message.channel !== 'iopub') { // RAWKERNEL: emit unhandled diff --git a/src/test/datascience/raw-kernel/rawKernel.unit.test.ts b/src/test/datascience/raw-kernel/rawKernel.unit.test.ts index dfb9fef06513..89a93dcff63b 100644 --- a/src/test/datascience/raw-kernel/rawKernel.unit.test.ts +++ b/src/test/datascience/raw-kernel/rawKernel.unit.test.ts @@ -158,20 +158,10 @@ suite('Data Science - RawKernel', () => { // Finally an idle message const iopubIdleMessage = buildStatusMessage('idle', rawKernel.clientId, future.msg.header); - - // Post the message mockJmpConnection.messageBack(iopubIdleMessage); // Last thing back is a reply message - const replyOptions: KernelMessage.IOptions = { - channel: 'shell', - session: rawKernel.clientId, - msgType: 'execute_reply', - content: { status: 'ok', execution_count: 1, payload: [], user_expressions: {} } - }; - const replyMessage = KernelMessage.createMessage(replyOptions); - replyMessage.parent_header = future.msg.header; - + const replyMessage = buildExecuteReplyMessage(rawKernel.clientId, future.msg.header); mockJmpConnection.messageBack(replyMessage); // Before we await for done we need to set up what we expect to see in our output @@ -307,9 +297,98 @@ suite('Data Science - RawKernel', () => { expect((messageIn.content as any).value).to.equal('input'); expect(messageIn.content.status).to.equal('ok'); }); + + // display_id can do some special handling with messages. Basically execute_reply or + // display_data messages can be tagged with a display_id then updated later by a + // update_display_data message + test('rawKernel displayid check', async () => { + const displayId = '1'; + await rawKernel.connect(connectInfo); + + // Check our status at the start + expect(rawKernel.status).to.equal('unknown'); + + // Create a future for an execute code request + const code = 'print("hello world")'; + const executeContent: KernelMessage.IExecuteRequestMsg['content'] = { + code + }; + const future = rawKernel.requestExecute(executeContent, true, undefined); + + // Next send an iopub display_data message with a displayId tag + const displayDataMessage = buildDisplayDataMessage(rawKernel.clientId, future.msg.header, displayId); + mockJmpConnection.messageBack(displayDataMessage); + + // Create a second future for another execute code request + const code2 = 'print("hello world 2")'; + const executeContent2: KernelMessage.IExecuteRequestMsg['content'] = { + code: code2 + }; + const future2 = rawKernel.requestExecute(executeContent2, true, undefined); + + // The second future also gets a display data message with the same id + const displayDataMessage2 = buildDisplayDataMessage(rawKernel.clientId, future2.msg.header, displayId); + mockJmpConnection.messageBack(displayDataMessage2); + + // Now send an iopub update_display_data with the same displayId tag + const uddOptions: KernelMessage.IOptions = { + channel: 'iopub', + session: rawKernel.clientId, + msgType: 'update_display_data', + parentHeader: future.msg.header, + content: { data: {}, metadata: {}, transient: { display_id: '1' } } + }; + const updateDDMessage = KernelMessage.createMessage(uddOptions); + mockJmpConnection.messageBack(updateDDMessage); + + // An idle message and a reply to finish things off for future one + const iopubIdleMessage = buildStatusMessage('idle', rawKernel.clientId, future.msg.header); + mockJmpConnection.messageBack(iopubIdleMessage); + + const replyMessage = buildExecuteReplyMessage(rawKernel.clientId, future.msg.header); + mockJmpConnection.messageBack(replyMessage); + + // An idle message and a reply to finish things off for future two + const iopubIdleMessage2 = buildStatusMessage('idle', rawKernel.clientId, future2.msg.header); + mockJmpConnection.messageBack(iopubIdleMessage2); + + const replyMessage2 = buildExecuteReplyMessage(rawKernel.clientId, future2.msg.header); + mockJmpConnection.messageBack(replyMessage2); + + // Validation here is that both futures have seen the update_display_data message + // not just one due to the display id + let futureSeen = false; + let future2Seen = false; + future.onIOPub = msg => { + if (msg.header.msg_id === updateDDMessage.header.msg_id) { + futureSeen = true; + } + }; + + future2.onIOPub = msg => { + if (msg.header.msg_id === updateDDMessage.header.msg_id) { + future2Seen = true; + } + }; + + await Promise.all([future.done, future2.done]); + expect(futureSeen).to.equal(true, 'Future did not see the update_display_data'); + expect(future2Seen).to.equal(true, 'Future2 did not see the update_display_data'); + }); }); }); +export function buildDisplayDataMessage(session: string, parentHeader: KernelMessage.IHeader, displayId?: string) { + const ddOptions: KernelMessage.IOptions = { + channel: 'iopub', + session, + msgType: 'display_data', + parentHeader, + content: { data: {}, metadata: {}, transient: displayId ? { display_id: '1' } : undefined } + }; + return KernelMessage.createMessage(ddOptions); +} + export function buildStatusMessage(status: Kernel.Status, session: string, parentHeader: KernelMessage.IHeader) { const iopubStatusOptions: KernelMessage.IOptions = { channel: 'iopub', From a2c5a6f1727ba29c519d66077164fe8202de1d92 Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Fri, 27 Mar 2020 08:18:24 -0700 Subject: [PATCH 011/725] add kernel experiment (#10820) Co-authored-by: Ian Huff --- experiments.json | 12 ++++++++++++ package.json | 2 ++ src/client/common/experimentGroups.ts | 6 ++++++ 3 files changed, 20 insertions(+) diff --git a/experiments.json b/experiments.json index c69b3551c210..52193c4b6d4d 100644 --- a/experiments.json +++ b/experiments.json @@ -113,6 +113,18 @@ "min": 0, "max": 100 }, + { + "name": "LocalZMQKernel - experiment", + "salt": "LocalZMQKernel", + "max": 0, + "min": 0 + }, + { + "name": "LocalZMQKernel - control", + "salt": "LocalZMQKernel", + "min": 0, + "max": 100 + }, { "name": "CollectLSRequestTiming - experiment", "salt": "CollectLSRequestTiming", diff --git a/package.json b/package.json index 88efde35687a..118b86919ddf 100644 --- a/package.json +++ b/package.json @@ -1556,6 +1556,7 @@ "Reload - experiment", "AA_testing - experiment", "WebHostNotebook - experiment", + "LocalZMQKernel - experiment", "UseTerminalToGetActivatedEnvVars - experiment", "CollectLSRequestTiming - experiment", "CollectNodeLSRequestTiming - experiment", @@ -1578,6 +1579,7 @@ "Reload - experiment", "AA_testing - experiment", "WebHostNotebook - experiment", + "LocalZMQKernel - experiment", "UseTerminalToGetActivatedEnvVars - experiment", "CollectLSRequestTiming - experiment", "CollectNodeLSRequestTiming - experiment", diff --git a/src/client/common/experimentGroups.ts b/src/client/common/experimentGroups.ts index 1bd0f149f4ea..c54e8f4446b1 100644 --- a/src/client/common/experimentGroups.ts +++ b/src/client/common/experimentGroups.ts @@ -42,6 +42,12 @@ export enum WebHostNotebook { experiment = 'WebHostNotebook - experiment' } +// Experiment to use a local ZMQ kernel connection as opposed to starting a Jupyter server locally +export enum LocalZMQKernel { + control = 'LocalZMQKernel - control', + experiment = 'LocalZMQKernel - experiment' +} + /** * Experiment to check whether to to use a terminal to generate the environment variables of activated environments. * From 5f2a19fbb384cb4ecd1376d84b6db779a1ad746c Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Wed, 15 Apr 2020 08:04:19 -0700 Subject: [PATCH 012/725] npm run prettier-fix --- .../datascience/jupyter/jupyterExecution.ts | 4 +- .../jupyter/kernels/kernelSelections.ts | 30 ++--- .../enchannel-zmq-backend-6/index.ts | 6 +- .../datascience/raw-kernel/rawKernel.ts | 10 +- .../datascience/dataScienceIocContainer.ts | 50 ++++---- src/test/datascience/execution.unit.test.ts | 110 +++++++++--------- ...eractiveWindowCommandListener.unit.test.ts | 18 +-- .../raw-kernel/rawFuture.unit.test.ts | 12 +- .../raw-kernel/rawKernel.functional.test.ts | 26 ++--- .../raw-kernel/rawKernel.unit.test.ts | 10 +- 10 files changed, 135 insertions(+), 141 deletions(-) diff --git a/src/client/datascience/jupyter/jupyterExecution.ts b/src/client/datascience/jupyter/jupyterExecution.ts index 997ac8739932..fabcaafc18af 100644 --- a/src/client/datascience/jupyter/jupyterExecution.ts +++ b/src/client/datascience/jupyter/jupyterExecution.ts @@ -59,7 +59,7 @@ export class JupyterExecutionBase implements IJupyterExecution { this.disposableRegistry.push(this); if (workspace) { - const disposable = workspace.onDidChangeConfiguration(e => { + const disposable = workspace.onDidChangeConfiguration((e) => { if (e.affectsConfiguration('python.dataScience', undefined)) { // When config changes happen, recreate our commands. this.onSettingsChanged(); @@ -307,7 +307,7 @@ export class JupyterExecutionBase implements IJupyterExecution { if (allowUI) { this.appShell .showErrorMessage(localize.DataScience.jupyterStartTimedout(), localize.Common.openOutputPanel()) - .then(selection => { + .then((selection) => { if (selection === localize.Common.openOutputPanel()) { this.jupyterOutputChannel.show(); } diff --git a/src/client/datascience/jupyter/kernels/kernelSelections.ts b/src/client/datascience/jupyter/kernels/kernelSelections.ts index a34ae38e0e77..f6d6fca67ce0 100644 --- a/src/client/datascience/jupyter/kernels/kernelSelections.ts +++ b/src/client/datascience/jupyter/kernels/kernelSelections.ts @@ -73,10 +73,10 @@ export class ActiveJupyterSessionKernelSelectionListProvider implements IKernelS this.sessionManager.getRunningSessions(), this.sessionManager.getKernelSpecs() ]); - const items = activeSessions.map(item => { + const items = activeSessions.map((item) => { const matchingSpec: Partial = - kernelSpecs.find(spec => spec.name === item.kernel.name) || {}; - const activeKernel = activeKernels.find(active => active.id === item.kernel.id) || {}; + kernelSpecs.find((spec) => spec.name === item.kernel.name) || {}; + const activeKernel = activeKernels.find((active) => active.id === item.kernel.id) || {}; // tslint:disable-next-line: no-object-literal-type-assertion return { ...item.kernel, @@ -86,10 +86,10 @@ export class ActiveJupyterSessionKernelSelectionListProvider implements IKernelS } as LiveKernelModel; }); return items - .filter(item => item.display_name || item.name) - .filter(item => 'lastActivityTime' in item && 'numberOfConnections' in item) - .filter(item => (item.language || '').toLowerCase() === PYTHON_LANGUAGE.toLowerCase()) - .map(item => getQuickPickItemForActiveKernel(item, this.pathUtils)); + .filter((item) => item.display_name || item.name) + .filter((item) => 'lastActivityTime' in item && 'numberOfConnections' in item) + .filter((item) => (item.language || '').toLowerCase() === PYTHON_LANGUAGE.toLowerCase()) + .map((item) => getQuickPickItemForActiveKernel(item, this.pathUtils)); } } @@ -112,8 +112,8 @@ export class InstalledJupyterKernelSelectionListProvider implements IKernelSelec ): Promise { const items = await this.kernelService.getKernelSpecs(this.sessionManager, cancelToken); return items - .filter(item => (item.language || '').toLowerCase() === PYTHON_LANGUAGE.toLowerCase()) - .map(item => getQuickPickItemForKernelSpec(item, this.pathUtils)); + .filter((item) => (item.language || '').toLowerCase() === PYTHON_LANGUAGE.toLowerCase()) + .map((item) => getQuickPickItemForKernelSpec(item, this.pathUtils)); } } @@ -132,7 +132,7 @@ export class InterpreterKernelSelectionListProvider implements IKernelSelectionL _cancelToken?: CancellationToken | undefined ): Promise { const items = await this.interpreterSelector.getSuggestions(resource); - return items.map(item => { + return items.map((item) => { return { ...item, // We don't want descriptions. @@ -190,7 +190,7 @@ export class KernelSelectionProvider { return [...liveKernels!, ...installedKernels!]; }; - const liveItems = getSelections().then(items => (this.localSuggestionsCache = items)); + const liveItems = getSelections().then((items) => (this.localSuggestionsCache = items)); // If we have someting in cache, return that, while fetching in the background. const cachedItems = this.remoteSuggestionsCache.length > 0 ? Promise.resolve(this.remoteSuggestionsCache) : liveItems; @@ -223,11 +223,11 @@ export class KernelSelectionProvider { let [installedKernels, interpreters] = await Promise.all([installedKernelsPromise, interpretersPromise]); interpreters = interpreters - .filter(item => { + .filter((item) => { // If the interpreter is registered as a kernel then don't inlcude it. if ( installedKernels.find( - installedKernel => + (installedKernel) => installedKernel.selection.kernelSpec?.display_name === item.selection.interpreter?.displayName && (this.fileSystem.arePathsSame( @@ -244,7 +244,7 @@ export class KernelSelectionProvider { } return true; }) - .map(item => { + .map((item) => { // We don't want descriptions. return { ...item, description: '' }; }); @@ -256,7 +256,7 @@ export class KernelSelectionProvider { return unifiedList; }; - const liveItems = getSelections().then(items => (this.localSuggestionsCache = items)); + const liveItems = getSelections().then((items) => (this.localSuggestionsCache = items)); // If we have someting in cache, return that, while fetching in the background. const cachedItems = this.localSuggestionsCache.length > 0 ? Promise.resolve(this.localSuggestionsCache) : liveItems; diff --git a/src/client/datascience/raw-kernel/enchannel-zmq-backend-6/index.ts b/src/client/datascience/raw-kernel/enchannel-zmq-backend-6/index.ts index b75b96f34548..0ae50893f0c5 100644 --- a/src/client/datascience/raw-kernel/enchannel-zmq-backend-6/index.ts +++ b/src/client/datascience/raw-kernel/enchannel-zmq-backend-6/index.ts @@ -146,7 +146,7 @@ class SocketEventEmitter extends Events.EventEmitter { // tslint:disable-next-line: no-floating-promises socket .receive() - .then(b => { + .then((b) => { this.emit('message', b); setTimeout(this.waitForReceive.bind(this, socket), 0); }) @@ -176,7 +176,7 @@ export const createMainChannelFromSockets = ( // The mega subject that encapsulates all the sockets as one multiplexed // stream const outgoingMessages = rxjs.Subscriber.create( - async message => { + async (message) => { // There's always a chance that a bad message is sent, we'll ignore it // instead of consuming it if (!message || !message.channel) { @@ -231,7 +231,7 @@ export const createMainChannelFromSockets = ( const incomingMessages: rxjs.Observable = rxjs .merge( // Form an Observable with each socket - ...Object.keys(sockets).map(name => { + ...Object.keys(sockets).map((name) => { // Wrap in something that will emit an event whenever a message is received. const socketEmitter = new SocketEventEmitter((sockets as any)[name]); return rxjs.fromEvent(socketEmitter, 'message').pipe( diff --git a/src/client/datascience/raw-kernel/rawKernel.ts b/src/client/datascience/raw-kernel/rawKernel.ts index cd940c1cacd9..f55626419a22 100644 --- a/src/client/datascience/raw-kernel/rawKernel.ts +++ b/src/client/datascience/raw-kernel/rawKernel.ts @@ -101,7 +101,7 @@ export class RawKernel implements Kernel.IKernel { public async connect(connectInfo: IJMPConnectionInfo) { await this.jmpConnection.connect(connectInfo); - this.jmpConnection.subscribe(message => { + this.jmpConnection.subscribe((message) => { this.msgIn(message); }); } @@ -247,7 +247,7 @@ export class RawKernel implements Kernel.IKernel { } // Dispose of all our outstanding futures - this.futures.forEach(future => { + this.futures.forEach((future) => { future.dispose(); }); this.futures.clear(); @@ -341,7 +341,7 @@ export class RawKernel implements Kernel.IKernel { return; } - displayIds.forEach(displayId => { + displayIds.forEach((displayId) => { const messageIds = this.displayIdToParentIds.get(displayId); if (messageIds) { const index = messageIds.indexOf(messageId); @@ -378,7 +378,7 @@ export class RawKernel implements Kernel.IKernel { // processing the next one return this.handleMessage(message); }) - .catch(error => { + .catch((error) => { traceError(error); }); } @@ -407,7 +407,7 @@ export class RawKernel implements Kernel.IKernel { // Now send it out to all the parents await Promise.all( - parentIds.map(async parentId => { + parentIds.map(async (parentId) => { const future = this.futures && this.futures.get(parentId); if (future) { await future.handleMessage(updateMessage); diff --git a/src/test/datascience/dataScienceIocContainer.ts b/src/test/datascience/dataScienceIocContainer.ts index d34972ec2349..1b8d83ba8d31 100644 --- a/src/test/datascience/dataScienceIocContainer.ts +++ b/src/test/datascience/dataScienceIocContainer.ts @@ -489,7 +489,7 @@ export class DataScienceIocContainer extends UnitTestIocContainer { const services = require('monaco-editor/esm/vs/editor/standalone/browser/standaloneServices') as any; if (services.StaticServices) { const keys = Object.keys(services.StaticServices); - keys.forEach(k => { + keys.forEach((k) => { const service = services.StaticServices[k] as any; if (service && service._value && service._value.dispose) { if (typeof service._value.dispose === 'function') { @@ -626,8 +626,8 @@ export class DataScienceIocContainer extends UnitTestIocContainer { ServerPreload ); const mockExtensionContext = TypeMoq.Mock.ofType(); - mockExtensionContext.setup(m => m.globalStoragePath).returns(() => os.tmpdir()); - mockExtensionContext.setup(m => m.extensionPath).returns(() => os.tmpdir()); + mockExtensionContext.setup((m) => m.globalStoragePath).returns(() => os.tmpdir()); + mockExtensionContext.setup((m) => m.extensionPath).returns(() => os.tmpdir()); this.serviceManager.addSingletonInstance(IExtensionContext, mockExtensionContext.object); const mockServerSelector = mock(JupyterServerSelector); @@ -836,10 +836,10 @@ export class DataScienceIocContainer extends UnitTestIocContainer { const configurationService = TypeMoq.Mock.ofType(); this.datascience = TypeMoq.Mock.ofType(); - configurationService.setup(c => c.getSettings(TypeMoq.It.isAny())).returns(this.getSettings.bind(this)); + configurationService.setup((c) => c.getSettings(TypeMoq.It.isAny())).returns(this.getSettings.bind(this)); const startTime = Date.now(); - this.datascience.setup(d => d.activationStartTime).returns(() => startTime); + this.datascience.setup((d) => d.activationStartTime).returns(() => startTime); this.serviceManager.addSingleton( IEnvironmentVariablesProvider, @@ -927,7 +927,7 @@ export class DataScienceIocContainer extends UnitTestIocContainer { } const interpreterDisplay = TypeMoq.Mock.ofType(); - interpreterDisplay.setup(i => i.refresh(TypeMoq.It.isAny())).returns(() => Promise.resolve()); + interpreterDisplay.setup((i) => i.refresh(TypeMoq.It.isAny())).returns(() => Promise.resolve()); // Create our jupyter mock if necessary if (this.shouldMockJupyter) { @@ -1052,9 +1052,9 @@ export class DataScienceIocContainer extends UnitTestIocContainer { // Don't use conda at all when mocking const condaService = TypeMoq.Mock.ofType(); this.serviceManager.addSingletonInstance(ICondaService, condaService.object); - condaService.setup(c => c.isCondaAvailable()).returns(() => Promise.resolve(false)); - condaService.setup(c => c.isCondaEnvironment(TypeMoq.It.isAny())).returns(() => Promise.resolve(false)); - condaService.setup(c => c.condaEnvironmentsFile).returns(() => undefined); + condaService.setup((c) => c.isCondaAvailable()).returns(() => Promise.resolve(false)); + condaService.setup((c) => c.isCondaEnvironment(TypeMoq.It.isAny())).returns(() => Promise.resolve(false)); + condaService.setup((c) => c.condaEnvironmentsFile).returns(() => undefined); this.serviceManager.addSingleton( IVirtualEnvironmentsSearchPathProvider, @@ -1111,23 +1111,23 @@ export class DataScienceIocContainer extends UnitTestIocContainer { } }; - appShell.setup(a => a.showErrorMessage(TypeMoq.It.isAnyString())).returns(() => Promise.resolve('')); + appShell.setup((a) => a.showErrorMessage(TypeMoq.It.isAnyString())).returns(() => Promise.resolve('')); appShell - .setup(a => a.showInformationMessage(TypeMoq.It.isAny(), TypeMoq.It.isAny())) + .setup((a) => a.showInformationMessage(TypeMoq.It.isAny(), TypeMoq.It.isAny())) .returns(() => Promise.resolve('')); appShell - .setup(a => a.showInformationMessage(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())) + .setup((a) => a.showInformationMessage(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())) .returns((_a1: string, a2: string, _a3: string) => Promise.resolve(a2)); appShell - .setup(a => + .setup((a) => a.showInformationMessage(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny()) ) .returns((_a1: string, a2: string, _a3: string, _a4: string) => Promise.resolve(a2)); appShell - .setup(a => a.showSaveDialog(TypeMoq.It.isAny())) + .setup((a) => a.showSaveDialog(TypeMoq.It.isAny())) .returns(() => Promise.resolve(Uri.file('test.ipynb'))); - appShell.setup(a => a.setStatusBarMessage(TypeMoq.It.isAny())).returns(() => dummyDisposable); - appShell.setup(a => a.showInputBox(TypeMoq.It.isAny())).returns(() => Promise.resolve('Python')); + appShell.setup((a) => a.setStatusBarMessage(TypeMoq.It.isAny())).returns(() => dummyDisposable); + appShell.setup((a) => a.showInputBox(TypeMoq.It.isAny())).returns(() => Promise.resolve('Python')); const interpreterManager = this.serviceContainer.get(IInterpreterService); interpreterManager.initialize(); @@ -1148,7 +1148,7 @@ export class DataScienceIocContainer extends UnitTestIocContainer { IExtensionSingleActivationService ); - await Promise.all(activationServices.map(a => a.activate())); + await Promise.all(activationServices.map((a) => a.activate())); // Then force our interpreter to be one that supports jupyter (unless in a mock state when we don't have to) if (!this.mockJupyter) { @@ -1236,9 +1236,9 @@ export class DataScienceIocContainer extends UnitTestIocContainer { return DataScienceIocContainer.jupyterInterpreters; } const list = await this.get(IInterpreterService).getInterpreters(undefined); - const promises = list.map(f => this.hasJupyter(f).then(b => (b ? f : undefined))); + const promises = list.map((f) => this.hasJupyter(f).then((b) => (b ? f : undefined))); const resolved = await Promise.all(promises); - DataScienceIocContainer.jupyterInterpreters = resolved.filter(r => r) as PythonInterpreter[]; + DataScienceIocContainer.jupyterInterpreters = resolved.filter((r) => r) as PythonInterpreter[]; return DataScienceIocContainer.jupyterInterpreters; } @@ -1249,7 +1249,7 @@ export class DataScienceIocContainer extends UnitTestIocContainer { } public addResourceToFolder(resource: Uri, folderPath: string) { - let folder = this.workspaceFolders.find(f => f.uri.fsPath === folderPath); + let folder = this.workspaceFolders.find((f) => f.uri.fsPath === folderPath); if (!folder) { folder = this.addWorkspaceFolder(folderPath); } @@ -1293,7 +1293,7 @@ export class DataScienceIocContainer extends UnitTestIocContainer { } if (this.extraListeners.length) { - this.extraListeners.forEach(e => e(msg.type, msg.payload)); + this.extraListeners.forEach((e) => e(msg.type, msg.payload)); } if (this.wrapperCreatedPromise && !this.wrapperCreatedPromise.resolved) { this.wrapperCreatedPromise.resolve(); @@ -1313,7 +1313,7 @@ export class DataScienceIocContainer extends UnitTestIocContainer { postMessage: noop as any, close: noop, updateCwd: noop as any, - asWebviewUri: uri => uri + asWebviewUri: (uri) => uri }); } } @@ -1333,7 +1333,7 @@ export class DataScienceIocContainer extends UnitTestIocContainer { private createWebPanel(): IWebPanel { const webPanel = mock(WebPanel); - when(webPanel.postMessage(anything())).thenCall(m => { + when(webPanel.postMessage(anything())).thenCall((m) => { // tslint:disable-next-line: no-require-imports const reactHelpers = require('./reactHelpers') as typeof import('./reactHelpers'); const message = reactHelpers.createMessageEvent(m); @@ -1359,7 +1359,7 @@ export class DataScienceIocContainer extends UnitTestIocContainer { // This needs to be async because we are being called in the ctor of the webpanel. It can't // handle some messages during the ctor. setTimeout(() => { - this.missedMessages.forEach(m => + this.missedMessages.forEach((m) => this.webPanelListener ? this.webPanelListener.onMessage(m.type, m.payload) : noop() ); }, 0); @@ -1478,7 +1478,7 @@ export class DataScienceIocContainer extends UnitTestIocContainer { private getWorkspaceFolder(uri: Resource): WorkspaceFolder | undefined { if (uri) { - return this.workspaceFolders.find(w => w.ownedResources.has(uri.toString())); + return this.workspaceFolders.find((w) => w.ownedResources.has(uri.toString())); } return undefined; } diff --git a/src/test/datascience/execution.unit.test.ts b/src/test/datascience/execution.unit.test.ts index cb378c0aed4b..1cfcb0fe6f2f 100644 --- a/src/test/datascience/execution.unit.test.ts +++ b/src/test/datascience/execution.unit.test.ts @@ -223,28 +223,28 @@ suite('Jupyter Execution', async () => { ) { if (module) { service - .setup(x => + .setup((x) => x.execModule( TypeMoq.It.isValue(module), - TypeMoq.It.is(a => argsMatch(args, a)), + TypeMoq.It.is((a) => argsMatch(args, a)), TypeMoq.It.isAny() ) ) .returns(() => result); const withModuleArgs = ['-m', module, ...args]; service - .setup(x => + .setup((x) => x.exec( - TypeMoq.It.is(a => argsMatch(withModuleArgs, a)), + TypeMoq.It.is((a) => argsMatch(withModuleArgs, a)), TypeMoq.It.isAny() ) ) .returns(() => result); } else { service - .setup(x => + .setup((x) => x.exec( - TypeMoq.It.is(a => argsMatch(args, a)), + TypeMoq.It.is((a) => argsMatch(args, a)), TypeMoq.It.isAny() ) ) @@ -259,28 +259,28 @@ suite('Jupyter Execution', async () => { result: () => Promise> ) { service - .setup(x => + .setup((x) => x.execModule( TypeMoq.It.isValue(module), - TypeMoq.It.is(a => argsMatch(args, a)), + TypeMoq.It.is((a) => argsMatch(args, a)), TypeMoq.It.isAny() ) ) .returns(result); const withModuleArgs = ['-m', module, ...args]; service - .setup(x => + .setup((x) => x.exec( - TypeMoq.It.is(a => argsMatch(withModuleArgs, a)), + TypeMoq.It.is((a) => argsMatch(withModuleArgs, a)), TypeMoq.It.isAny() ) ) .returns(result); service - .setup(x => + .setup((x) => x.execModule( TypeMoq.It.isValue(module), - TypeMoq.It.is(a => argsMatch(args, a)), + TypeMoq.It.is((a) => argsMatch(args, a)), TypeMoq.It.isAny() ) ) @@ -296,9 +296,9 @@ suite('Jupyter Execution', async () => { ) { const result: ObservableExecutionResult = { proc: undefined, - out: new Observable>(subscriber => { - stderr.forEach(s => subscriber.next({ source: 'stderr', out: s })); - stdout.forEach(s => subscriber.next({ source: 'stderr', out: s })); + out: new Observable>((subscriber) => { + stderr.forEach((s) => subscriber.next({ source: 'stderr', out: s })); + stdout.forEach((s) => subscriber.next({ source: 'stderr', out: s })); }), dispose: () => { noop(); @@ -306,19 +306,19 @@ suite('Jupyter Execution', async () => { }; service - .setup(x => + .setup((x) => x.execModuleObservable( TypeMoq.It.isValue(module), - TypeMoq.It.is(a => argsMatch(args, a)), + TypeMoq.It.is((a) => argsMatch(args, a)), TypeMoq.It.isAny() ) ) .returns(() => result); const withModuleArgs = ['-m', module, ...args]; service - .setup(x => + .setup((x) => x.execObservable( - TypeMoq.It.is(a => argsMatch(withModuleArgs, a)), + TypeMoq.It.is((a) => argsMatch(withModuleArgs, a)), TypeMoq.It.isAny() ) ) @@ -332,10 +332,10 @@ suite('Jupyter Execution', async () => { result: Promise> ) { service - .setup(x => + .setup((x) => x.exec( TypeMoq.It.isValue(file), - TypeMoq.It.is(a => argsMatch(args, a)), + TypeMoq.It.is((a) => argsMatch(args, a)), TypeMoq.It.isAny() ) ) @@ -349,10 +349,10 @@ suite('Jupyter Execution', async () => { result: () => Promise> ) { service - .setup(x => + .setup((x) => x.exec( TypeMoq.It.isValue(file), - TypeMoq.It.is(a => argsMatch(args, a)), + TypeMoq.It.is((a) => argsMatch(args, a)), TypeMoq.It.isAny() ) ) @@ -368,9 +368,9 @@ suite('Jupyter Execution', async () => { ) { const result: ObservableExecutionResult = { proc: undefined, - out: new Observable>(subscriber => { - stderr.forEach(s => subscriber.next({ source: 'stderr', out: s })); - stdout.forEach(s => subscriber.next({ source: 'stderr', out: s })); + out: new Observable>((subscriber) => { + stderr.forEach((s) => subscriber.next({ source: 'stderr', out: s })); + stdout.forEach((s) => subscriber.next({ source: 'stderr', out: s })); }), dispose: () => { noop(); @@ -378,10 +378,10 @@ suite('Jupyter Execution', async () => { }; service - .setup(x => + .setup((x) => x.execObservable( TypeMoq.It.isValue(file), - TypeMoq.It.is(a => argsMatch(args, a)), + TypeMoq.It.is((a) => argsMatch(args, a)), TypeMoq.It.isAny() ) ) @@ -390,7 +390,7 @@ suite('Jupyter Execution', async () => { function createKernelSpecs(specs: { name: string; resourceDir: string }[]): Record { const models: Record = {}; - specs.forEach(spec => { + specs.forEach((spec) => { models[spec.name] = { resource_dir: spec.resourceDir, spec: { @@ -411,7 +411,7 @@ suite('Jupyter Execution', async () => { setupPythonService(service, 'jupyter', ['nbconvert', '--version'], Promise.resolve({ stdout: '1.1.1.1' })); setupPythonService(service, 'jupyter', ['notebook', '--version'], Promise.resolve({ stdout: '1.1.1.1' })); setupPythonService(service, 'jupyter', ['kernelspec', '--version'], Promise.resolve({ stdout: '1.1.1.1' })); - service.setup(x => x.getInterpreterInformation()).returns(() => Promise.resolve(workingPython)); + service.setup((x) => x.getInterpreterInformation()).returns(() => Promise.resolve(workingPython)); // Don't mind the goofy path here. It's supposed to not find the item. It's just testing the internal regex works setupPythonServiceWithFunc(service, 'jupyter', ['kernelspec', 'list', '--json'], () => { @@ -499,7 +499,7 @@ suite('Jupyter Execution', async () => { ) { setupPythonService(service, 'jupyter', ['notebook', '--version'], Promise.resolve({ stdout: '1.1.1.1' })); setupPythonService(service, 'jupyter', ['kernelspec', '--version'], Promise.resolve({ stdout: '1.1.1.1' })); - service.setup(x => x.getInterpreterInformation()).returns(() => Promise.resolve(missingKernelPython)); + service.setup((x) => x.getInterpreterInformation()).returns(() => Promise.resolve(missingKernelPython)); const kernelSpecs = createKernelSpecs([{ name: 'working', resourceDir: path.dirname(workingKernelSpec) }]); setupPythonService( service, @@ -537,11 +537,11 @@ suite('Jupyter Execution', async () => { function setupMissingNotebookPythonService(service: TypeMoq.IMock) { service - .setup(x => x.execModule(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())) - .returns(_v => { + .setup((x) => x.execModule(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())) + .returns((_v) => { return Promise.reject('cant exec'); }); - service.setup(x => x.getInterpreterInformation()).returns(() => Promise.resolve(missingNotebookPython)); + service.setup((x) => x.getInterpreterInformation()).returns(() => Promise.resolve(missingNotebookPython)); } function setupWorkingProcessService(service: TypeMoq.IMock, notebookStdErr?: string[]) { @@ -768,7 +768,7 @@ suite('Jupyter Execution', async () => { when(interpreterService.getInterpreterDetails(match('/foo/bar/python.exe'))).thenResolve(workingPython); // Mockito is stupid. Matchers have to use literals. when(interpreterService.getInterpreterDetails(match('/foo/baz/python.exe'))).thenResolve(missingKernelPython); when(interpreterService.getInterpreterDetails(match('/bar/baz/python.exe'))).thenResolve(missingNotebookPython); - when(interpreterService.getInterpreterDetails(argThat(o => !o.includes || !o.includes('python')))).thenReject( + when(interpreterService.getInterpreterDetails(argThat((o) => !o.includes || !o.includes('python')))).thenReject( ('Unknown interpreter' as any) as Error ); if (runInDocker) { @@ -787,33 +787,33 @@ suite('Jupyter Execution', async () => { setupWorkingProcessService(processService, notebookStdErr); setupMissingKernelProcessService(processService, notebookStdErr); setupPathProcessService(jupyterOnPath, processService, notebookStdErr); - when(executionFactory.create(argThat(o => o.pythonPath && o.pythonPath === workingPython.path))).thenResolve( + when(executionFactory.create(argThat((o) => o.pythonPath && o.pythonPath === workingPython.path))).thenResolve( workingService.object ); when( - executionFactory.create(argThat(o => o.pythonPath && o.pythonPath === missingKernelPython.path)) + executionFactory.create(argThat((o) => o.pythonPath && o.pythonPath === missingKernelPython.path)) ).thenResolve(missingKernelService.object); when( - executionFactory.create(argThat(o => o.pythonPath && o.pythonPath === missingNotebookPython.path)) + executionFactory.create(argThat((o) => o.pythonPath && o.pythonPath === missingNotebookPython.path)) ).thenResolve(missingNotebookService.object); when( - executionFactory.create(argThat(o => o.pythonPath && o.pythonPath === missingNotebookPython2.path)) + executionFactory.create(argThat((o) => o.pythonPath && o.pythonPath === missingNotebookPython2.path)) ).thenResolve(missingNotebookService2.object); when( - executionFactory.createDaemon(argThat(o => o.pythonPath && o.pythonPath === workingPython.path)) + executionFactory.createDaemon(argThat((o) => o.pythonPath && o.pythonPath === workingPython.path)) ).thenResolve(workingService.object); when( - executionFactory.createDaemon(argThat(o => o.pythonPath && o.pythonPath === missingKernelPython.path)) + executionFactory.createDaemon(argThat((o) => o.pythonPath && o.pythonPath === missingKernelPython.path)) ).thenResolve(missingKernelService.object); when( - executionFactory.createDaemon(argThat(o => o.pythonPath && o.pythonPath === missingNotebookPython.path)) + executionFactory.createDaemon(argThat((o) => o.pythonPath && o.pythonPath === missingNotebookPython.path)) ).thenResolve(missingNotebookService.object); when( - executionFactory.createDaemon(argThat(o => o.pythonPath && o.pythonPath === missingNotebookPython2.path)) + executionFactory.createDaemon(argThat((o) => o.pythonPath && o.pythonPath === missingNotebookPython2.path)) ).thenResolve(missingNotebookService2.object); let activeService = workingService; @@ -824,26 +824,26 @@ suite('Jupyter Execution', async () => { } else if (activeInterpreter === missingNotebookPython2) { activeService = missingNotebookService2; } - when(executionFactory.create(argThat(o => !o || !o.pythonPath))).thenResolve(activeService.object); + when(executionFactory.create(argThat((o) => !o || !o.pythonPath))).thenResolve(activeService.object); when( - executionFactory.createActivatedEnvironment(argThat(o => !o || o.interpreter === activeInterpreter)) + executionFactory.createActivatedEnvironment(argThat((o) => !o || o.interpreter === activeInterpreter)) ).thenResolve(activeService.object); when( - executionFactory.createActivatedEnvironment(argThat(o => o && o.interpreter.path === workingPython.path)) + executionFactory.createActivatedEnvironment(argThat((o) => o && o.interpreter.path === workingPython.path)) ).thenResolve(workingService.object); when( executionFactory.createActivatedEnvironment( - argThat(o => o && o.interpreter.path === missingKernelPython.path) + argThat((o) => o && o.interpreter.path === missingKernelPython.path) ) ).thenResolve(missingKernelService.object); when( executionFactory.createActivatedEnvironment( - argThat(o => o && o.interpreter.path === missingNotebookPython.path) + argThat((o) => o && o.interpreter.path === missingNotebookPython.path) ) ).thenResolve(missingNotebookService.object); when( executionFactory.createActivatedEnvironment( - argThat(o => o && o.interpreter.path === missingNotebookPython2.path) + argThat((o) => o && o.interpreter.path === missingNotebookPython2.path) ) ).thenResolve(missingNotebookService2.object); when(processServiceFactory.create()).thenResolve(processService.object); @@ -860,9 +860,7 @@ suite('Jupyter Execution', async () => { when(application.withProgress(anything(), anything())).thenCall( (_, cb: (_: any, token: any) => Promise) => { return new Promise((resolve, reject) => { - cb({ report: noop }, new CancellationTokenSource().token) - .then(resolve) - .catch(reject); + cb({ report: noop }, new CancellationTokenSource().token).then(resolve).catch(reject); }); } ); @@ -1153,9 +1151,7 @@ suite('Jupyter Execution', async () => { when(application.withProgress(anything(), anything())).thenCall( (_, cb: (_: any, token: any) => Promise) => { return new Promise((resolve, reject) => { - cb({ report: noop }, progressCancellation.token) - .then(resolve) - .catch(reject); + cb({ report: noop }, progressCancellation.token).then(resolve).catch(reject); }); } ); @@ -1179,9 +1175,7 @@ suite('Jupyter Execution', async () => { when(application.withProgress(anything(), anything())).thenCall( (_, cb: (_: any, token: any) => Promise) => { return new Promise((resolve, reject) => { - cb({ report: noop }, progressCancellation.token) - .then(resolve) - .catch(reject); + cb({ report: noop }, progressCancellation.token).then(resolve).catch(reject); }); } ); diff --git a/src/test/datascience/interactiveWindowCommandListener.unit.test.ts b/src/test/datascience/interactiveWindowCommandListener.unit.test.ts index 8823d51f5855..3bc761654562 100644 --- a/src/test/datascience/interactiveWindowCommandListener.unit.test.ts +++ b/src/test/datascience/interactiveWindowCommandListener.unit.test.ts @@ -106,7 +106,7 @@ suite('Interactive window command listener', async () => { // Setup defaults when(interpreterService.onDidChangeInterpreter).thenReturn(dummyEvent.event); - when(interpreterService.getInterpreterDetails(argThat(o => !o.includes || !o.includes('python')))).thenReject( + when(interpreterService.getInterpreterDetails(argThat((o) => !o.includes || !o.includes('python')))).thenReject( ('Unknown interpreter' as any) as Error ); @@ -159,7 +159,7 @@ suite('Interactive window command listener', async () => { when( fileSystem.writeFile( anything(), - argThat(o => { + argThat((o) => { lastFileContents = o; return true; }) @@ -221,7 +221,7 @@ suite('Interactive window command listener', async () => { test('Import', async () => { createCommandListener(); - when(applicationShell.showOpenDialog(argThat(o => o.openLabel && o.openLabel.includes('Import')))).thenReturn( + when(applicationShell.showOpenDialog(argThat((o) => o.openLabel && o.openLabel.includes('Import')))).thenReturn( Promise.resolve([Uri.file('foo')]) ); await commandManager.executeCommand(Commands.ImportNotebook, undefined, undefined); @@ -236,7 +236,7 @@ suite('Interactive window command listener', async () => { createCommandListener(); const doc = await documentManager.openTextDocument('bar.ipynb'); await documentManager.showTextDocument(doc); - when(applicationShell.showSaveDialog(argThat(o => o.saveLabel && o.saveLabel.includes('Export')))).thenReturn( + when(applicationShell.showSaveDialog(argThat((o) => o.saveLabel && o.saveLabel.includes('Export')))).thenReturn( Promise.resolve(Uri.file('foo')) ); when(applicationShell.showInformationMessage(anything(), anything())).thenReturn(Promise.resolve('moo')); @@ -263,10 +263,10 @@ suite('Interactive window command listener', async () => { when(jupyterExecution.connectToNotebookServer(anything(), anything())).thenResolve(server.object); const notebook = createTypeMoq('jupyter notebook'); server - .setup(s => s.createNotebook(TypeMoq.It.isAny(), TypeMoq.It.isAny())) + .setup((s) => s.createNotebook(TypeMoq.It.isAny(), TypeMoq.It.isAny())) .returns(() => Promise.resolve(notebook.object)); notebook - .setup(n => + .setup((n) => n.execute( TypeMoq.It.isAny(), TypeMoq.It.isAny(), @@ -279,7 +279,7 @@ suite('Interactive window command listener', async () => { return Promise.resolve(generateCells(undefined, 'a=1', 'bar.py', 0, false, uuid())); }); - when(applicationShell.showSaveDialog(argThat(o => o.saveLabel && o.saveLabel.includes('Export')))).thenReturn( + when(applicationShell.showSaveDialog(argThat((o) => o.saveLabel && o.saveLabel.includes('Export')))).thenReturn( Promise.resolve(Uri.file('foo')) ); when(applicationShell.showInformationMessage(anything(), anything())).thenReturn(Promise.resolve('moo')); @@ -301,7 +301,7 @@ suite('Interactive window command listener', async () => { }); test('Export skipped on no file', async () => { createCommandListener(); - when(applicationShell.showSaveDialog(argThat(o => o.saveLabel && o.saveLabel.includes('Export')))).thenReturn( + when(applicationShell.showSaveDialog(argThat((o) => o.saveLabel && o.saveLabel.includes('Export')))).thenReturn( Promise.resolve(Uri.file('foo')) ); await commandManager.executeCommand(Commands.ExportFileAndOutputAsNotebook, Uri.file('bar.ipynb')); @@ -311,7 +311,7 @@ suite('Interactive window command listener', async () => { createCommandListener(); const doc = await documentManager.openTextDocument('bar.ipynb'); await documentManager.showTextDocument(doc); - when(applicationShell.showSaveDialog(argThat(o => o.saveLabel && o.saveLabel.includes('Export')))).thenReturn( + when(applicationShell.showSaveDialog(argThat((o) => o.saveLabel && o.saveLabel.includes('Export')))).thenReturn( Promise.resolve(Uri.file('foo')) ); await commandManager.executeCommand(Commands.ExportFileAsNotebook, undefined, undefined); diff --git a/src/test/datascience/raw-kernel/rawFuture.unit.test.ts b/src/test/datascience/raw-kernel/rawFuture.unit.test.ts index dec3cd3d8033..6a9da3e642f5 100644 --- a/src/test/datascience/raw-kernel/rawFuture.unit.test.ts +++ b/src/test/datascience/raw-kernel/rawFuture.unit.test.ts @@ -29,17 +29,17 @@ suite('Data Science - RawFuture', () => { test('RawFuture dispose', async () => { // Set up some handlers - rawFuture.onReply = _msg => { + rawFuture.onReply = (_msg) => { noop(); }; - rawFuture.onIOPub = _msg => { + rawFuture.onIOPub = (_msg) => { noop(); }; - rawFuture.onStdin = _msg => { + rawFuture.onStdin = (_msg) => { noop(); }; - rawFuture.done.catch(reason => { + rawFuture.done.catch((reason) => { const error = reason as Error; expect(error.message).to.equal('Disposed Future'); }); @@ -103,7 +103,7 @@ suite('Data Science - RawFuture', () => { replyMessage.parent_header = executeMessage.header; // Verify that the reply message matches the one we sent - rawFuture.onReply = msg => { + rawFuture.onReply = (msg) => { expect(msg.header.msg_id).to.equal(replyMessage.header.msg_id); }; @@ -128,7 +128,7 @@ suite('Data Science - RawFuture', () => { ioPubMessage.parent_header = executeMessage.header; // Verify that the iopub message matches the one we sent - rawFuture.onIOPub = msg => { + rawFuture.onIOPub = (msg) => { expect(msg.header.msg_id).to.equal(ioPubMessage.header.msg_id); }; diff --git a/src/test/datascience/raw-kernel/rawKernel.functional.test.ts b/src/test/datascience/raw-kernel/rawKernel.functional.test.ts index ff6a8060d3ba..4f542727bbd4 100644 --- a/src/test/datascience/raw-kernel/rawKernel.functional.test.ts +++ b/src/test/datascience/raw-kernel/rawKernel.functional.test.ts @@ -35,7 +35,7 @@ suite('DataScience raw kernel tests', () => { kernel_name: 'python3', version: 5.1 }; - setup(async function() { + setup(async function () { ioc = new DataScienceIocContainer(); ioc.registerDataScienceTypes(); await ioc.activate(); @@ -75,10 +75,10 @@ suite('DataScience raw kernel tests', () => { throwOnStdErr: false }); kernelProcResult.out.subscribe( - out => { + (out) => { console.log(out.out); }, - error => { + (error) => { console.error(error); }, () => { @@ -87,7 +87,7 @@ suite('DataScience raw kernel tests', () => { ); sessionId = uuid(); await enchannelConnection.connect(connectionInfo); - messageObservable = new Observable(subscriber => { + messageObservable = new Observable((subscriber) => { enchannelConnection.subscribe(subscriber.next.bind(subscriber)); }); } @@ -186,7 +186,7 @@ suite('DataScience raw kernel tests', () => { } let foundReply = false; let foundIdle = false; - const subscr = messageObservable.subscribe(m => { + const subscr = messageObservable.subscribe((m) => { replies.push(m); if (m.header.msg_type === 'status') { foundIdle = (m.content as any).execution_state === 'idle'; @@ -203,7 +203,7 @@ suite('DataScience raw kernel tests', () => { } }); enchannelConnection.sendMessage(message); - return waiter.promise.then(m => { + return waiter.promise.then((m) => { subscr.unsubscribe(); return m; }); @@ -212,40 +212,40 @@ suite('DataScience raw kernel tests', () => { test('Basic connection', async () => { const replies = await sendMessage(createShutdownMessage()); assert.ok( - replies.find(r => r.header.msg_type === 'shutdown_reply'), + replies.find((r) => r.header.msg_type === 'shutdown_reply'), 'Reply not sent for shutdown' ); }); test('Basic request', async () => { const replies = await sendMessage(createExecutionMessage('a=1\na')); - const executeResult = replies.find(r => r.header.msg_type === 'execute_result'); + const executeResult = replies.find((r) => r.header.msg_type === 'execute_result'); assert.ok(executeResult, 'Result not found'); assert.equal((executeResult?.content as any).data['text/plain'], '1', 'Results were not computed'); }); test('Multiple requests', async () => { let replies = await sendMessage(createExecutionMessage('a=1\na')); - let executeResult = replies.find(r => r.header.msg_type === 'execute_result'); + let executeResult = replies.find((r) => r.header.msg_type === 'execute_result'); assert.ok(executeResult, 'Result not found'); replies = await sendMessage(createExecutionMessage('a=2\na')); - executeResult = replies.find(r => r.header.msg_type === 'execute_result'); + executeResult = replies.find((r) => r.header.msg_type === 'execute_result'); assert.ok(executeResult, 'Result 2 not found'); assert.equal((executeResult?.content as any).data['text/plain'], '2', 'Results were not computed'); replies = await sendMessage(createInspectMessage('a')); - const inspectResult = replies.find(r => r.header.msg_type === 'inspect_reply'); + const inspectResult = replies.find((r) => r.header.msg_type === 'inspect_reply'); assert.ok(inspectResult, 'Inspect result not found'); assert.ok((inspectResult?.content as any).data['text/plain'], 'Inspect reply was not computed'); }); test('Startup and shutdown', async () => { let replies = await sendMessage(createExecutionMessage('a=1\na')); - let executeResult = replies.find(r => r.header.msg_type === 'execute_result'); + let executeResult = replies.find((r) => r.header.msg_type === 'execute_result'); assert.ok(executeResult, 'Result not found'); await disconnectFromKernel(); await connectToKernel(57418); replies = await sendMessage(createExecutionMessage('a=1\na')); - executeResult = replies.find(r => r.header.msg_type === 'execute_result'); + executeResult = replies.find((r) => r.header.msg_type === 'execute_result'); assert.ok(executeResult, 'Result not found'); }); }); diff --git a/src/test/datascience/raw-kernel/rawKernel.unit.test.ts b/src/test/datascience/raw-kernel/rawKernel.unit.test.ts index 89a93dcff63b..1c447041210f 100644 --- a/src/test/datascience/raw-kernel/rawKernel.unit.test.ts +++ b/src/test/datascience/raw-kernel/rawKernel.unit.test.ts @@ -88,7 +88,7 @@ suite('Data Science - RawKernel', () => { code }; const future = rawKernel.requestExecute(executeContent, true, undefined); - future.done.catch(reason => { + future.done.catch((reason) => { const error = reason as Error; expect(error.message).to.equal('Disposed Future'); }); @@ -169,7 +169,7 @@ suite('Data Science - RawKernel', () => { // Check our IOPub Messages const iopubMessages = [iopubBusyMessage, iopubExecuteInputMessage, iopubStreamMessage, iopubIdleMessage]; let iopubHit = 0; - future.onIOPub = msg => { + future.onIOPub = (msg) => { const targetMsg = iopubMessages[iopubHit]; expect(msg.header.msg_id).to.equal(targetMsg.header.msg_id); iopubHit = iopubHit + 1; @@ -178,7 +178,7 @@ suite('Data Science - RawKernel', () => { // Check our reply messages const replyMessages = [replyMessage]; let replyHit = 0; - future.onReply = msg => { + future.onReply = (msg) => { const targetMsg = replyMessages[replyHit]; expect(msg.header.msg_id).to.equal(targetMsg.header.msg_id); replyHit = replyHit + 1; @@ -359,13 +359,13 @@ suite('Data Science - RawKernel', () => { // not just one due to the display id let futureSeen = false; let future2Seen = false; - future.onIOPub = msg => { + future.onIOPub = (msg) => { if (msg.header.msg_id === updateDDMessage.header.msg_id) { futureSeen = true; } }; - future2.onIOPub = msg => { + future2.onIOPub = (msg) => { if (msg.header.msg_id === updateDDMessage.header.msg_id) { future2Seen = true; } From f1dd9e508f25e4b2e8e38e9e9fb9004496ccdc0a Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Wed, 1 Apr 2020 12:45:15 -0700 Subject: [PATCH 013/725] Notebook refactor to prep for raw kernel work (#10903) --- .../interactive-common/interactiveBase.ts | 107 ++-- .../interactive-common/notebookProvider.ts | 201 +------ .../notebookServerProvider.ts | 202 +++++++ .../interactive-window/interactiveWindow.ts | 8 +- .../ipyWidgetScriptSourceProvider.ts | 10 +- .../datascience/jupyter/jupyterConnection.ts | 510 +++++++++--------- .../datascience/jupyter/jupyterDebugger.ts | 8 +- .../datascience/jupyter/jupyterNotebook.ts | 61 ++- .../datascience/jupyter/jupyterUtils.ts | 11 +- .../jupyter/kernels/kernelSwitcher.ts | 10 +- .../jupyter/liveshare/guestJupyterNotebook.ts | 14 +- .../jupyter/liveshare/guestJupyterServer.ts | 2 +- .../jupyter/liveshare/hostJupyterExecution.ts | 340 ++++++------ .../jupyter/liveshare/hostJupyterNotebook.ts | 9 +- .../jupyter/liveshare/hostJupyterServer.ts | 1 - .../datascience/jupyter/serverPreload.ts | 10 +- src/client/datascience/serviceRegistry.ts | 3 + src/client/datascience/types.ts | 62 ++- .../datascience/dataScienceIocContainer.ts | 3 + .../notebookProvider.unit.test.ts | 125 +++++ .../notebookServerProvider.unit.test.ts | 95 ++++ ...cdnWidgetScriptSourceProvider.unit.test.ts | 3 + ...ipyWidgetScriptSourceProvider.unit.test.ts | 3 + .../kernels/kernelSwitcher.unit.test.ts | 21 +- src/test/datascience/mockJupyterNotebook.ts | 12 +- src/test/datascience/mockJupyterServer.ts | 4 +- .../datascience/notebook.functional.test.ts | 3 +- 27 files changed, 1087 insertions(+), 751 deletions(-) create mode 100644 src/client/datascience/interactive-common/notebookServerProvider.ts create mode 100644 src/test/datascience/interactive-common/notebookProvider.unit.test.ts create mode 100644 src/test/datascience/interactive-common/notebookServerProvider.unit.test.ts diff --git a/src/client/datascience/interactive-common/interactiveBase.ts b/src/client/datascience/interactive-common/interactiveBase.ts index 7112f7806f1d..9fcbbe04191c 100644 --- a/src/client/datascience/interactive-common/interactiveBase.ts +++ b/src/client/datascience/interactive-common/interactiveBase.ts @@ -73,7 +73,6 @@ import { CellState, ICell, ICodeCssGenerator, - IConnection, IDataScienceErrorHandler, IDataViewerProvider, IInteractiveBase, @@ -89,7 +88,7 @@ import { INotebook, INotebookExporter, INotebookProvider, - INotebookServer, + INotebookProviderConnection, InterruptResult, IStatusProvider, IThemeFinder, @@ -119,7 +118,7 @@ export abstract class InteractiveBase extends WebViewHost = new EventEmitter(); - private serverAndNotebookPromise: Promise | undefined; + private connectionAndNotebookPromise: Promise | undefined; private notebookPromise: Promise | undefined; private setDarkPromise: Deferred | undefined; @@ -195,12 +194,12 @@ export abstract class InteractiveBase extends WebViewHost { // Verify a server that matches us hasn't started already - this.checkForServerStart().ignoreErrors(); + this.checkForNotebookProviderConnection().ignoreErrors(); // Show our web panel. return super.show(true); @@ -502,7 +501,7 @@ export abstract class InteractiveBase extends WebViewHost; protected async clearResult(id: string): Promise { - await this.ensureServerAndNotebook(); + await this.ensureConnectionAndNotebook(); if (this._notebook) { this._notebook.clear(id); } @@ -564,7 +563,7 @@ export abstract class InteractiveBase extends WebViewHost { - if (!this.serverAndNotebookPromise) { - this.serverAndNotebookPromise = this.ensureServerAndNotebookImpl(); + protected async ensureConnectionAndNotebook(): Promise { + if (!this.connectionAndNotebookPromise) { + this.connectionAndNotebookPromise = this.ensureConnectionAndNotebookImpl(); } try { - await this.serverAndNotebookPromise; + await this.connectionAndNotebookPromise; } catch (e) { // Reset the load promise. Don't want to keep hitting the same error - this.serverAndNotebookPromise = undefined; + this.connectionAndNotebookPromise = undefined; throw e; } } @@ -808,13 +807,13 @@ export abstract class InteractiveBase extends WebViewHost { + private async ensureConnectionAndNotebookImpl(): Promise { // Make sure we're loaded first. try { traceInfo('Waiting for jupyter server and web panel ...'); - const server = await this.notebookProvider.getOrCreateServer({ getOnly: false, disableUI: false }); - if (server) { - await this.ensureNotebook(server); + const serverConnection = await this.notebookProvider.connect({ getOnly: false, disableUI: false }); + if (serverConnection) { + await this.ensureNotebook(serverConnection); } } catch (exc) { // We should dispose ourselves if the load fails. Othewise the user @@ -978,7 +977,7 @@ export abstract class InteractiveBase extends WebViewHost { // Finish either of our notebook promises - if (this.serverAndNotebookPromise) { - await this.serverAndNotebookPromise; - this.serverAndNotebookPromise = undefined; + if (this.connectionAndNotebookPromise) { + await this.connectionAndNotebookPromise; + this.connectionAndNotebookPromise = undefined; } if (this.notebookPromise) { await this.notebookPromise; @@ -1011,19 +1010,17 @@ export abstract class InteractiveBase extends WebViewHost { + protected async ensureNotebook(serverConnection: INotebookProviderConnection): Promise { if (!this.notebookPromise) { - this.notebookPromise = this.ensureNotebookImpl(server); + this.notebookPromise = this.ensureNotebookImpl(serverConnection); } try { await this.notebookPromise; @@ -1035,7 +1032,7 @@ export abstract class InteractiveBase extends WebViewHost { + private async createNotebook(serverConnection: INotebookProviderConnection): Promise { let notebook: INotebook | undefined; while (!notebook) { const [resource, identity, metadata] = await Promise.all([ @@ -1049,7 +1046,7 @@ export abstract class InteractiveBase extends WebViewHost { + private async ensureNotebookImpl(serverConnection: INotebookProviderConnection): Promise { // Create a new notebook if we need to. if (!this._notebook) { // While waiting make the notebook look busy this.postMessage(InteractiveWindowMessages.UpdateKernel, { jupyterServerStatus: ServerStatus.Busy, - localizedUri: this.getServerUri(server), + localizedUri: this.getServerUri(serverConnection), displayName: '' }).ignoreErrors(); - this._notebook = await this.createNotebook(server); + this._notebook = await this.createNotebook(serverConnection); // If that works notify the UI and listen to status changes. if (this._notebook && this._notebook.identity) { @@ -1135,15 +1127,13 @@ export abstract class InteractiveBase extends WebViewHost { - // See if a server already started or not - const server = await this.notebookProvider.getOrCreateServer({ getOnly: true }); + private async checkForNotebookProviderConnection(): Promise { + // Check to see if we are already connected to our provider + const providerConnection = await this.notebookProvider.connect({ getOnly: true }); - // This means a server that matches our options has started already. Use - // it to ensure we have a notebook to run. - if (server) { + if (providerConnection) { try { - await this.ensureNotebook(server); + await this.ensureNotebook(providerConnection); } catch (e) { this.errorHandler.handleError(e).ignoreErrors(); } @@ -1158,10 +1148,10 @@ export abstract class InteractiveBase extends WebViewHost 0 ? `?token=${connInfo.token}` : ''; - const urlString = `${connInfo.baseUrl}${tokenString}`; - - return `${localize.DataScience.sysInfoURILabel()}${urlString}`; + private generateConnectionInfoString(connInfo: INotebookProviderConnection | undefined): string { + return connInfo?.displayName || ''; } private async requestVariables(args: IJupyterVariablesRequest): Promise { diff --git a/src/client/datascience/interactive-common/notebookProvider.ts b/src/client/datascience/interactive-common/notebookProvider.ts index 13b896d3de39..230bed2d71e9 100644 --- a/src/client/datascience/interactive-common/notebookProvider.ts +++ b/src/client/datascience/interactive-common/notebookProvider.ts @@ -4,53 +4,35 @@ 'use strict'; import { inject, injectable } from 'inversify'; -import { ConfigurationTarget, EventEmitter, Uri } from 'vscode'; -import { IApplicationShell } from '../../common/application/types'; -import { CancellationError } from '../../common/cancellation'; -import { traceInfo } from '../../common/logger'; +import { EventEmitter, Uri } from 'vscode'; import { IFileSystem } from '../../common/platform/types'; -import { IConfigurationService, IDisposableRegistry } from '../../common/types'; -import * as localize from '../../common/utils/localize'; +import { IDisposableRegistry } from '../../common/types'; import { noop } from '../../common/utils/misc'; -import { IInterpreterService } from '../../interpreter/contracts'; -import { sendTelemetryEvent } from '../../telemetry'; -import { Identifiers, Settings, Telemetry } from '../constants'; -import { JupyterInstallError } from '../jupyter/jupyterInstallError'; -import { JupyterSelfCertsError } from '../jupyter/jupyterSelfCertsError'; -import { JupyterZMQBinariesNotFoundError } from '../jupyter/jupyterZMQBinariesNotFoundError'; -import { ProgressReporter } from '../progress/progressReporter'; import { + ConnectNotebookProviderOptions, GetNotebookOptions, - GetServerOptions, IInteractiveWindowProvider, - IJupyterExecution, INotebook, INotebookEditor, INotebookEditorProvider, INotebookProvider, - INotebookServer, - INotebookServerOptions + INotebookProviderConnection, + INotebookServerProvider } from '../types'; @injectable() export class NotebookProvider implements INotebookProvider { private readonly notebooks = new Map>(); - private serverPromise: Promise | undefined; - private allowingUI = false; private _notebookCreated = new EventEmitter<{ identity: Uri; notebook: INotebook }>(); public get activeNotebooks() { return [...this.notebooks.values()]; } constructor( - @inject(ProgressReporter) private readonly progressReporter: ProgressReporter, - @inject(IConfigurationService) private readonly configuration: IConfigurationService, @inject(IFileSystem) private readonly fs: IFileSystem, @inject(INotebookEditorProvider) private readonly editorProvider: INotebookEditorProvider, @inject(IInteractiveWindowProvider) private readonly interactiveWindowProvider: IInteractiveWindowProvider, @inject(IDisposableRegistry) disposables: IDisposableRegistry, - @inject(IJupyterExecution) private readonly jupyterExecution: IJupyterExecution, - @inject(IApplicationShell) private readonly applicationShell: IApplicationShell, - @inject(IInterpreterService) private readonly interpreterService: IInterpreterService + @inject(INotebookServerProvider) private readonly serverProvider: INotebookServerProvider ) { disposables.push(editorProvider.onDidCloseNotebookEditor(this.onDidCloseNotebookEditor, this)); disposables.push( @@ -61,21 +43,26 @@ export class NotebookProvider implements INotebookProvider { return this._notebookCreated.event; } - public async getOrCreateServer(options: GetServerOptions): Promise { - const serverOptions = this.getNotebookServerOptions(); + // Disconnect from the specified provider + public async disconnect(options: ConnectNotebookProviderOptions): Promise { + const server = await this.serverProvider.getOrCreateServer(options); - // If we are just fetching or only want to create for local, see if exists - if (options.getOnly || (options.localOnly && serverOptions.uri)) { - return this.jupyterExecution.getServer(serverOptions); - } else { - // Otherwise create a new server - return this.createServer(options); - } + return server?.dispose(); + } + + // Attempt to connect to our server provider, and if we do, return the connection info + public async connect(options: ConnectNotebookProviderOptions): Promise { + const server = await this.serverProvider.getOrCreateServer(options); + + return server?.getConnectionInfo(); } public async getOrCreateNotebook(options: GetNotebookOptions): Promise { // Make sure we have a server - const server = await this.getOrCreateServer({ getOnly: options.getOnly, disableUI: options.disableUI }); + const server = await this.serverProvider.getOrCreateServer({ + getOnly: options.getOnly, + disableUI: options.disableUI + }); if (server) { // We could have multiple native editors opened for the same file/model. const notebook = await server.getNotebook(options.identity); @@ -113,152 +100,6 @@ export class NotebookProvider implements INotebookProvider { } } - private async createServer(options: GetServerOptions): Promise { - // When we finally try to create a server, update our flag indicating if we're going to allow UI or not. This - // allows the server to be attempted without a UI, but a future request can come in and use the same startup - this.allowingUI = options.disableUI ? this.allowingUI : true; - - if (!this.serverPromise) { - // Start a server - this.serverPromise = this.startServer(); - } - try { - return await this.serverPromise; - } catch (e) { - // Don't cache the error - this.serverPromise = undefined; - throw e; - } - } - - private async startServer(): Promise { - const serverOptions = this.getNotebookServerOptions(); - - traceInfo(`Checking for server existence.`); - - // Status depends upon if we're about to connect to existing server or not. - const progressReporter = this.allowingUI - ? (await this.jupyterExecution.getServer(serverOptions)) - ? this.progressReporter.createProgressIndicator(localize.DataScience.connectingToJupyter()) - : this.progressReporter.createProgressIndicator(localize.DataScience.startingJupyter()) - : undefined; - - // Check to see if we support ipykernel or not - try { - traceInfo(`Checking for server usability.`); - - const usable = await this.checkUsable(serverOptions); - if (!usable) { - traceInfo('Server not usable (should ask for install now)'); - // Indicate failing. - throw new JupyterInstallError( - localize.DataScience.jupyterNotSupported().format(await this.jupyterExecution.getNotebookError()), - localize.DataScience.pythonInteractiveHelpLink() - ); - } - // Then actually start the server - traceInfo(`Starting notebook server.`); - const result = await this.jupyterExecution.connectToNotebookServer(serverOptions, progressReporter?.token); - traceInfo(`Server started.`); - return result; - } catch (e) { - progressReporter?.dispose(); // NOSONAR - // If user cancelled, then do nothing. - if (progressReporter && progressReporter.token.isCancellationRequested && e instanceof CancellationError) { - return; - } - - // Also tell jupyter execution to reset its search. Otherwise we've just cached - // the failure there - await this.jupyterExecution.refreshCommands(); - - if (e instanceof JupyterSelfCertsError) { - // On a self cert error, warn the user and ask if they want to change the setting - const enableOption: string = localize.DataScience.jupyterSelfCertEnable(); - const closeOption: string = localize.DataScience.jupyterSelfCertClose(); - this.applicationShell - .showErrorMessage( - localize.DataScience.jupyterSelfCertFail().format(e.message), - enableOption, - closeOption - ) - .then((value) => { - if (value === enableOption) { - sendTelemetryEvent(Telemetry.SelfCertsMessageEnabled); - this.configuration - .updateSetting( - 'dataScience.allowUnauthorizedRemoteConnection', - true, - undefined, - ConfigurationTarget.Workspace - ) - .ignoreErrors(); - } else if (value === closeOption) { - sendTelemetryEvent(Telemetry.SelfCertsMessageClose); - } - }); - throw e; - } else { - throw e; - } - } finally { - progressReporter?.dispose(); // NOSONAR - } - } - - private async checkUsable(options: INotebookServerOptions): Promise { - try { - if (options && !options.uri) { - const usableInterpreter = await this.jupyterExecution.getUsableJupyterPython(); - return usableInterpreter ? true : false; - } else { - return true; - } - } catch (e) { - if (e instanceof JupyterZMQBinariesNotFoundError) { - throw e; - } - const activeInterpreter = await this.interpreterService.getActiveInterpreter(undefined); - // Can't find a usable interpreter, show the error. - if (activeInterpreter) { - const displayName = activeInterpreter.displayName - ? activeInterpreter.displayName - : activeInterpreter.path; - throw new Error( - localize.DataScience.jupyterNotSupportedBecauseOfEnvironment().format(displayName, e.toString()) - ); - } else { - throw new JupyterInstallError( - localize.DataScience.jupyterNotSupported().format(await this.jupyterExecution.getNotebookError()), - localize.DataScience.pythonInteractiveHelpLink() - ); - } - } - } - - private getNotebookServerOptions(): INotebookServerOptions { - // Since there's one server per session, don't use a resource to figure out these settings - const settings = this.configuration.getSettings(undefined); - let serverURI: string | undefined = settings.datascience.jupyterServerURI; - const useDefaultConfig: boolean | undefined = settings.datascience.useDefaultConfigForJupyter; - - // For the local case pass in our URI as undefined, that way connect doesn't have to check the setting - if (serverURI.toLowerCase() === Settings.JupyterServerLocalLaunch) { - serverURI = undefined; - } - - return { - uri: serverURI, - skipUsingDefaultConfig: !useDefaultConfig, - purpose: Identifiers.HistoryPurpose, - allowUI: this.allowUI.bind(this) - }; - } - - private allowUI(): boolean { - return this.allowingUI; - } - private async onDidCloseNotebookEditor(editor: INotebookEditor) { // First find all notebooks associated with this editor (ipynb file). const editors = this.editorProvider.editors.filter( diff --git a/src/client/datascience/interactive-common/notebookServerProvider.ts b/src/client/datascience/interactive-common/notebookServerProvider.ts new file mode 100644 index 000000000000..52d5dec5bcd8 --- /dev/null +++ b/src/client/datascience/interactive-common/notebookServerProvider.ts @@ -0,0 +1,202 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +import { inject, injectable } from 'inversify'; +import { ConfigurationTarget, EventEmitter, Uri } from 'vscode'; +import { IApplicationShell } from '../../common/application/types'; +import { CancellationError } from '../../common/cancellation'; +import { traceInfo } from '../../common/logger'; +import { IConfigurationService } from '../../common/types'; +import * as localize from '../../common/utils/localize'; +import { IInterpreterService } from '../../interpreter/contracts'; +import { sendTelemetryEvent } from '../../telemetry'; +import { Identifiers, Settings, Telemetry } from '../constants'; +import { JupyterInstallError } from '../jupyter/jupyterInstallError'; +import { JupyterSelfCertsError } from '../jupyter/jupyterSelfCertsError'; +import { JupyterZMQBinariesNotFoundError } from '../jupyter/jupyterZMQBinariesNotFoundError'; +import { ProgressReporter } from '../progress/progressReporter'; +import { + GetServerOptions, + IJupyterExecution, + INotebook, + INotebookServer, + INotebookServerOptions, + INotebookServerProvider +} from '../types'; + +@injectable() +export class NotebookServerProvider implements INotebookServerProvider { + private serverPromise: Promise | undefined; + private allowingUI = false; + private _notebookCreated = new EventEmitter<{ identity: Uri; notebook: INotebook }>(); + constructor( + @inject(ProgressReporter) private readonly progressReporter: ProgressReporter, + @inject(IConfigurationService) private readonly configuration: IConfigurationService, + @inject(IJupyterExecution) private readonly jupyterExecution: IJupyterExecution, + @inject(IApplicationShell) private readonly applicationShell: IApplicationShell, + @inject(IInterpreterService) private readonly interpreterService: IInterpreterService + ) {} + public get onNotebookCreated() { + return this._notebookCreated.event; + } + + public async getOrCreateServer(options: GetServerOptions): Promise { + const serverOptions = this.getNotebookServerOptions(); + + // If we are just fetching or only want to create for local, see if exists + if (options.getOnly || (options.localOnly && serverOptions.uri)) { + return this.jupyterExecution.getServer(serverOptions); + } else { + // Otherwise create a new server + return this.createServer(options); + } + } + + private async createServer(options: GetServerOptions): Promise { + // When we finally try to create a server, update our flag indicating if we're going to allow UI or not. This + // allows the server to be attempted without a UI, but a future request can come in and use the same startup + this.allowingUI = options.disableUI ? this.allowingUI : true; + + if (!this.serverPromise) { + // Start a server + this.serverPromise = this.startServer(); + } + try { + return await this.serverPromise; + } catch (e) { + // Don't cache the error + this.serverPromise = undefined; + throw e; + } + } + + private async startServer(): Promise { + const serverOptions = this.getNotebookServerOptions(); + + traceInfo(`Checking for server existence.`); + + // Status depends upon if we're about to connect to existing server or not. + const progressReporter = this.allowingUI + ? (await this.jupyterExecution.getServer(serverOptions)) + ? this.progressReporter.createProgressIndicator(localize.DataScience.connectingToJupyter()) + : this.progressReporter.createProgressIndicator(localize.DataScience.startingJupyter()) + : undefined; + + // Check to see if we support ipykernel or not + try { + traceInfo(`Checking for server usability.`); + + const usable = await this.checkUsable(serverOptions); + if (!usable) { + traceInfo('Server not usable (should ask for install now)'); + // Indicate failing. + throw new JupyterInstallError( + localize.DataScience.jupyterNotSupported().format(await this.jupyterExecution.getNotebookError()), + localize.DataScience.pythonInteractiveHelpLink() + ); + } + // Then actually start the server + traceInfo(`Starting notebook server.`); + const result = await this.jupyterExecution.connectToNotebookServer(serverOptions, progressReporter?.token); + traceInfo(`Server started.`); + return result; + } catch (e) { + progressReporter?.dispose(); // NOSONAR + // If user cancelled, then do nothing. + if (progressReporter && progressReporter.token.isCancellationRequested && e instanceof CancellationError) { + return; + } + + // Also tell jupyter execution to reset its search. Otherwise we've just cached + // the failure there + await this.jupyterExecution.refreshCommands(); + + if (e instanceof JupyterSelfCertsError) { + // On a self cert error, warn the user and ask if they want to change the setting + const enableOption: string = localize.DataScience.jupyterSelfCertEnable(); + const closeOption: string = localize.DataScience.jupyterSelfCertClose(); + this.applicationShell + .showErrorMessage( + localize.DataScience.jupyterSelfCertFail().format(e.message), + enableOption, + closeOption + ) + .then((value) => { + if (value === enableOption) { + sendTelemetryEvent(Telemetry.SelfCertsMessageEnabled); + this.configuration + .updateSetting( + 'dataScience.allowUnauthorizedRemoteConnection', + true, + undefined, + ConfigurationTarget.Workspace + ) + .ignoreErrors(); + } else if (value === closeOption) { + sendTelemetryEvent(Telemetry.SelfCertsMessageClose); + } + }); + throw e; + } else { + throw e; + } + } finally { + progressReporter?.dispose(); // NOSONAR + } + } + + private async checkUsable(options: INotebookServerOptions): Promise { + try { + if (options && !options.uri) { + const usableInterpreter = await this.jupyterExecution.getUsableJupyterPython(); + return usableInterpreter ? true : false; + } else { + return true; + } + } catch (e) { + if (e instanceof JupyterZMQBinariesNotFoundError) { + throw e; + } + const activeInterpreter = await this.interpreterService.getActiveInterpreter(undefined); + // Can't find a usable interpreter, show the error. + if (activeInterpreter) { + const displayName = activeInterpreter.displayName + ? activeInterpreter.displayName + : activeInterpreter.path; + throw new Error( + localize.DataScience.jupyterNotSupportedBecauseOfEnvironment().format(displayName, e.toString()) + ); + } else { + throw new JupyterInstallError( + localize.DataScience.jupyterNotSupported().format(await this.jupyterExecution.getNotebookError()), + localize.DataScience.pythonInteractiveHelpLink() + ); + } + } + } + + private getNotebookServerOptions(): INotebookServerOptions { + // Since there's one server per session, don't use a resource to figure out these settings + const settings = this.configuration.getSettings(undefined); + let serverURI: string | undefined = settings.datascience.jupyterServerURI; + const useDefaultConfig: boolean | undefined = settings.datascience.useDefaultConfigForJupyter; + + // For the local case pass in our URI as undefined, that way connect doesn't have to check the setting + if (serverURI.toLowerCase() === Settings.JupyterServerLocalLaunch) { + serverURI = undefined; + } + + return { + uri: serverURI, + skipUsingDefaultConfig: !useDefaultConfig, + purpose: Identifiers.HistoryPurpose, + allowUI: this.allowUI.bind(this) + }; + } + + private allowUI(): boolean { + return this.allowingUI; + } +} diff --git a/src/client/datascience/interactive-window/interactiveWindow.ts b/src/client/datascience/interactive-window/interactiveWindow.ts index 56b84e0bfafa..56956015a48a 100644 --- a/src/client/datascience/interactive-window/interactiveWindow.ts +++ b/src/client/datascience/interactive-window/interactiveWindow.ts @@ -150,7 +150,7 @@ export class InteractiveWindow extends InteractiveBase implements IInteractiveWi sendTelemetryEvent(Telemetry.OpenedInteractiveWindow); // Start the server as soon as we open - this.ensureServerAndNotebook().ignoreErrors(); + this.ensureConnectionAndNotebook().ignoreErrors(); } public dispose() { @@ -172,7 +172,7 @@ export class InteractiveWindow extends InteractiveBase implements IInteractiveWi await this.loadWebPanel(this.lastFile ? path.dirname(this.lastFile) : process.cwd()); // Make sure we're loaded first. InteractiveWindow doesn't makes sense without an active server. - await this.ensureServerAndNotebook(); + await this.ensureConnectionAndNotebook(); // Make sure we have at least the initial sys info await this.addSysInfo(SysInfoReason.Start); @@ -345,7 +345,7 @@ export class InteractiveWindow extends InteractiveBase implements IInteractiveWi protected async closeBecauseOfFailure(_exc: Error): Promise { this.dispose(); } - protected ensureServerAndNotebook(): Promise { + protected ensureConnectionAndNotebook(): Promise { // Keep track of users who have used interactive window in a worksapce folder. // To be used if/when changing workflows related to startup of jupyter. if (!this.trackedJupyterStart) { @@ -353,7 +353,7 @@ export class InteractiveWindow extends InteractiveBase implements IInteractiveWi const store = this.stateFactory.createGlobalPersistentState('INTERACTIVE_WINDOW_USED', false); store.updateValue(true).ignoreErrors(); } - return super.ensureServerAndNotebook(); + return super.ensureConnectionAndNotebook(); } private async addOrDebugCode(code: string, file: string, line: number, debug: boolean): Promise { diff --git a/src/client/datascience/ipywidgets/ipyWidgetScriptSourceProvider.ts b/src/client/datascience/ipywidgets/ipyWidgetScriptSourceProvider.ts index b9ecc802ba94..054fdd569b04 100644 --- a/src/client/datascience/ipywidgets/ipyWidgetScriptSourceProvider.ts +++ b/src/client/datascience/ipywidgets/ipyWidgetScriptSourceProvider.ts @@ -20,7 +20,7 @@ import { Common, DataScience } from '../../common/utils/localize'; import { IInterpreterService } from '../../interpreter/contracts'; import { sendTelemetryEvent } from '../../telemetry'; import { Telemetry } from '../constants'; -import { ILocalResourceUriConverter, INotebook } from '../types'; +import { IConnection, ILocalResourceUriConverter, INotebook } from '../types'; import { CDNWidgetScriptSourceProvider } from './cdnWidgetScriptSourceProvider'; import { LocalWidgetScriptSourceProvider } from './localWidgetScriptSourceProvider'; import { RemoteWidgetScriptSourceProvider } from './remoteWidgetScriptSourceProvider'; @@ -124,7 +124,7 @@ export class IPyWidgetScriptSourceProvider implements IWidgetScriptSourceProvide if (this.configuredScriptSources.length > 0) { scriptProviders.push(new CDNWidgetScriptSourceProvider(this.configurationSettings, this.httpClient)); } - if (this.notebook.connection.localLaunch) { + if (this.notebook.connection && this.notebook.connection.localLaunch) { scriptProviders.push( new LocalWidgetScriptSourceProvider( this.notebook, @@ -134,7 +134,11 @@ export class IPyWidgetScriptSourceProvider implements IWidgetScriptSourceProvide ) ); } else { - scriptProviders.push(new RemoteWidgetScriptSourceProvider(this.notebook.connection, this.httpClient)); + if (this.notebook.connection) { + scriptProviders.push( + new RemoteWidgetScriptSourceProvider(this.notebook.connection as IConnection, this.httpClient) + ); + } } this.scriptProviders = scriptProviders; diff --git a/src/client/datascience/jupyter/jupyterConnection.ts b/src/client/datascience/jupyter/jupyterConnection.ts index 304840e9023f..8a0138baf813 100644 --- a/src/client/datascience/jupyter/jupyterConnection.ts +++ b/src/client/datascience/jupyter/jupyterConnection.ts @@ -1,249 +1,261 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. -'use strict'; -import '../../common/extensions'; - -import { ChildProcess } from 'child_process'; -import { CancellationToken, Disposable, Event, EventEmitter } from 'vscode'; -import { Cancellation, CancellationError } from '../../common/cancellation'; -import { traceInfo, traceWarning } from '../../common/logger'; -import { IFileSystem } from '../../common/platform/types'; -import { ObservableExecutionResult, Output } from '../../common/process/types'; -import { IConfigurationService, IDisposable } from '../../common/types'; -import { createDeferred, Deferred } from '../../common/utils/async'; -import * as localize from '../../common/utils/localize'; -import { IServiceContainer } from '../../ioc/types'; -import { RegExpValues } from '../constants'; -import { IConnection } from '../types'; -import { JupyterConnectError } from './jupyterConnectError'; - -// tslint:disable-next-line:no-require-imports no-var-requires no-any -const namedRegexp = require('named-js-regexp'); -const urlMatcher = namedRegexp(RegExpValues.UrlPatternRegEx); - -export type JupyterServerInfo = { - base_url: string; - notebook_dir: string; - hostname: string; - password: boolean; - pid: number; - port: number; - secure: boolean; - token: string; - url: string; -}; - -export class JupyterConnectionWaiter implements IDisposable { - private startPromise: Deferred; - private launchTimeout: NodeJS.Timer | number; - private configService: IConfigurationService; - private fileSystem: IFileSystem; - private stderr: string[] = []; - private connectionDisposed = false; - - constructor( - private readonly launchResult: ObservableExecutionResult, - private readonly notebookDir: string, - private readonly getServerInfo: (cancelToken?: CancellationToken) => Promise, - serviceContainer: IServiceContainer, - private readonly cancelToken?: CancellationToken - ) { - this.configService = serviceContainer.get(IConfigurationService); - this.fileSystem = serviceContainer.get(IFileSystem); - - // Cancel our start promise if a cancellation occurs - if (cancelToken) { - cancelToken.onCancellationRequested(() => this.startPromise.reject(new CancellationError())); - } - - // Setup our start promise - this.startPromise = createDeferred(); - - // We want to reject our Jupyter connection after a specific timeout - const settings = this.configService.getSettings(undefined); - const jupyterLaunchTimeout = settings.datascience.jupyterLaunchTimeout; - - this.launchTimeout = setTimeout(() => { - this.launchTimedOut(); - }, jupyterLaunchTimeout); - - // Listen for crashes - let exitCode = '0'; - if (launchResult.proc) { - launchResult.proc.on('exit', (c) => (exitCode = c ? c.toString() : '0')); - } - let stderr = ''; - // Listen on stderr for its connection information - launchResult.out.subscribe( - (output: Output) => { - if (output.source === 'stderr') { - stderr += output.out; - this.stderr.push(output.out); - this.extractConnectionInformation(stderr); - } else { - this.output(output.out); - } - }, - (e) => this.rejectStartPromise(e.message), - // If the process dies, we can't extract connection information. - () => this.rejectStartPromise(localize.DataScience.jupyterServerCrashed().format(exitCode)) - ); - } - public dispose() { - // tslint:disable-next-line: no-any - clearTimeout(this.launchTimeout as any); - } - - public waitForConnection(): Promise { - return this.startPromise.promise; - } - - private createConnection(baseUrl: string, token: string, hostName: string, processDisposable: Disposable) { - // tslint:disable-next-line: no-use-before-declare - return new JupyterConnection(baseUrl, token, hostName, processDisposable, this.launchResult.proc); - } - - // tslint:disable-next-line:no-any - private output = (data: any) => { - if (!this.connectionDisposed) { - traceInfo(data.toString('utf8')); - } - }; - - // From a list of jupyter server infos try to find the matching jupyter that we launched - // tslint:disable-next-line:no-any - private getJupyterURL(serverInfos: JupyterServerInfo[] | undefined, data: any) { - if (serverInfos && serverInfos.length > 0 && !this.startPromise.completed) { - const matchInfo = serverInfos.find((info) => - this.fileSystem.arePathsSame(this.notebookDir, info.notebook_dir) - ); - if (matchInfo) { - const url = matchInfo.url; - const token = matchInfo.token; - const host = matchInfo.hostname; - this.resolveStartPromise(url, token, host); - } - } - // At this point we failed to get the server info or a matching server via the python code, so fall back to - // our URL parse - if (!this.startPromise.completed) { - this.getJupyterURLFromString(data); - } - } - - // tslint:disable-next-line:no-any - private getJupyterURLFromString(data: any) { - // tslint:disable-next-line:no-any - const urlMatch = urlMatcher.exec(data) as any; - const groups = urlMatch.groups() as RegExpValues.IUrlPatternGroupType; - if (urlMatch && !this.startPromise.completed && groups && (groups.LOCAL || groups.IP)) { - // Rebuild the URI from our group hits - const host = groups.LOCAL ? groups.LOCAL : groups.IP; - const uriString = `${groups.PREFIX}${host}${groups.REST}`; - - // URL is not being found for some reason. Pull it in forcefully - // tslint:disable-next-line:no-require-imports - const URL = require('url').URL; - let url: URL; - try { - url = new URL(uriString); - } catch (err) { - // Failed to parse the url either via server infos or the string - this.rejectStartPromise(localize.DataScience.jupyterLaunchNoURL()); - return; - } - - // Here we parsed the URL correctly - this.resolveStartPromise( - `${url.protocol}//${url.host}${url.pathname}`, - `${url.searchParams.get('token')}`, - url.hostname - ); - } - } - - // tslint:disable-next-line:no-any - private extractConnectionInformation = (data: any) => { - this.output(data); - - const httpMatch = RegExpValues.HttpPattern.exec(data); - - if (httpMatch && this.notebookDir && this.startPromise && !this.startPromise.completed && this.getServerInfo) { - // .then so that we can keep from pushing aync up to the subscribed observable function - this.getServerInfo(this.cancelToken) - .then((serverInfos) => this.getJupyterURL(serverInfos, data)) - .catch((ex) => traceWarning('Failed to get server info', ex)); - } - - // Sometimes jupyter will return a 403 error. Not sure why. We used - // to fail on this, but it looks like jupyter works with this error in place. - }; - - private launchTimedOut = () => { - if (!this.startPromise.completed) { - this.rejectStartPromise(localize.DataScience.jupyterLaunchTimedOut()); - } - }; - - private resolveStartPromise = (baseUrl: string, token: string, hostName: string) => { - // tslint:disable-next-line: no-any - clearTimeout(this.launchTimeout as any); - if (!this.startPromise.rejected) { - const connection = this.createConnection(baseUrl, token, hostName, this.launchResult); - const origDispose = connection.dispose.bind(connection); - connection.dispose = () => { - // Stop listening when we disconnect - this.connectionDisposed = true; - return origDispose(); - }; - this.startPromise.resolve(connection); - } - }; - - // tslint:disable-next-line:no-any - private rejectStartPromise = (message: string) => { - // tslint:disable-next-line: no-any - clearTimeout(this.launchTimeout as any); - if (!this.startPromise.resolved) { - this.startPromise.reject( - Cancellation.isCanceled(this.cancelToken) - ? new CancellationError() - : new JupyterConnectError(message, this.stderr.join('\n')) - ); - } - }; -} - -// Represents an active connection to a running jupyter notebook -class JupyterConnection implements IConnection { - public readonly localLaunch: boolean = true; - public localProcExitCode: number | undefined; - private eventEmitter: EventEmitter = new EventEmitter(); - constructor( - public readonly baseUrl: string, - public readonly token: string, - public readonly hostName: string, - private readonly disposable: Disposable, - childProc: ChildProcess | undefined - ) { - // If the local process exits, set our exit code and fire our event - if (childProc) { - childProc.on('exit', (c) => { - // Our code expects the exit code to be of type `number` or `undefined`. - const code = typeof c === 'number' ? c : undefined; - this.localProcExitCode = code; - this.eventEmitter.fire(code); - }); - } - } - - public get disconnected(): Event { - return this.eventEmitter.event; - } - - public dispose() { - if (this.disposable) { - this.disposable.dispose(); - } - } -} +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +'use strict'; +import '../../common/extensions'; + +import { ChildProcess } from 'child_process'; +import { CancellationToken, Disposable, Event, EventEmitter } from 'vscode'; +import { Cancellation, CancellationError } from '../../common/cancellation'; +import { traceInfo, traceWarning } from '../../common/logger'; +import { IFileSystem } from '../../common/platform/types'; +import { ObservableExecutionResult, Output } from '../../common/process/types'; +import { IConfigurationService, IDisposable } from '../../common/types'; +import { createDeferred, Deferred } from '../../common/utils/async'; +import * as localize from '../../common/utils/localize'; +import { IServiceContainer } from '../../ioc/types'; +import { RegExpValues } from '../constants'; +import { IConnection } from '../types'; +import { JupyterConnectError } from './jupyterConnectError'; + +// tslint:disable-next-line:no-require-imports no-var-requires no-any +const namedRegexp = require('named-js-regexp'); +const urlMatcher = namedRegexp(RegExpValues.UrlPatternRegEx); + +export type JupyterServerInfo = { + base_url: string; + notebook_dir: string; + hostname: string; + password: boolean; + pid: number; + port: number; + secure: boolean; + token: string; + url: string; +}; + +export class JupyterConnectionWaiter implements IDisposable { + private startPromise: Deferred; + private launchTimeout: NodeJS.Timer | number; + private configService: IConfigurationService; + private fileSystem: IFileSystem; + private stderr: string[] = []; + private connectionDisposed = false; + + constructor( + private readonly launchResult: ObservableExecutionResult, + private readonly notebookDir: string, + private readonly getServerInfo: (cancelToken?: CancellationToken) => Promise, + serviceContainer: IServiceContainer, + private readonly cancelToken?: CancellationToken + ) { + this.configService = serviceContainer.get(IConfigurationService); + this.fileSystem = serviceContainer.get(IFileSystem); + + // Cancel our start promise if a cancellation occurs + if (cancelToken) { + cancelToken.onCancellationRequested(() => this.startPromise.reject(new CancellationError())); + } + + // Setup our start promise + this.startPromise = createDeferred(); + + // We want to reject our Jupyter connection after a specific timeout + const settings = this.configService.getSettings(undefined); + const jupyterLaunchTimeout = settings.datascience.jupyterLaunchTimeout; + + this.launchTimeout = setTimeout(() => { + this.launchTimedOut(); + }, jupyterLaunchTimeout); + + // Listen for crashes + let exitCode = '0'; + if (launchResult.proc) { + launchResult.proc.on('exit', (c) => (exitCode = c ? c.toString() : '0')); + } + let stderr = ''; + // Listen on stderr for its connection information + launchResult.out.subscribe( + (output: Output) => { + if (output.source === 'stderr') { + stderr += output.out; + this.stderr.push(output.out); + this.extractConnectionInformation(stderr); + } else { + this.output(output.out); + } + }, + (e) => this.rejectStartPromise(e.message), + // If the process dies, we can't extract connection information. + () => this.rejectStartPromise(localize.DataScience.jupyterServerCrashed().format(exitCode)) + ); + } + public dispose() { + // tslint:disable-next-line: no-any + clearTimeout(this.launchTimeout as any); + } + + public waitForConnection(): Promise { + return this.startPromise.promise; + } + + private createConnection(baseUrl: string, token: string, hostName: string, processDisposable: Disposable) { + // tslint:disable-next-line: no-use-before-declare + return new JupyterConnection(baseUrl, token, hostName, processDisposable, this.launchResult.proc); + } + + // tslint:disable-next-line:no-any + private output = (data: any) => { + if (!this.connectionDisposed) { + traceInfo(data.toString('utf8')); + } + }; + + // From a list of jupyter server infos try to find the matching jupyter that we launched + // tslint:disable-next-line:no-any + private getJupyterURL(serverInfos: JupyterServerInfo[] | undefined, data: any) { + if (serverInfos && serverInfos.length > 0 && !this.startPromise.completed) { + const matchInfo = serverInfos.find((info) => + this.fileSystem.arePathsSame(this.notebookDir, info.notebook_dir) + ); + if (matchInfo) { + const url = matchInfo.url; + const token = matchInfo.token; + const host = matchInfo.hostname; + this.resolveStartPromise(url, token, host); + } + } + // At this point we failed to get the server info or a matching server via the python code, so fall back to + // our URL parse + if (!this.startPromise.completed) { + this.getJupyterURLFromString(data); + } + } + + // tslint:disable-next-line:no-any + private getJupyterURLFromString(data: any) { + // tslint:disable-next-line:no-any + const urlMatch = urlMatcher.exec(data) as any; + const groups = urlMatch.groups() as RegExpValues.IUrlPatternGroupType; + if (urlMatch && !this.startPromise.completed && groups && (groups.LOCAL || groups.IP)) { + // Rebuild the URI from our group hits + const host = groups.LOCAL ? groups.LOCAL : groups.IP; + const uriString = `${groups.PREFIX}${host}${groups.REST}`; + + // URL is not being found for some reason. Pull it in forcefully + // tslint:disable-next-line:no-require-imports + const URL = require('url').URL; + let url: URL; + try { + url = new URL(uriString); + } catch (err) { + // Failed to parse the url either via server infos or the string + this.rejectStartPromise(localize.DataScience.jupyterLaunchNoURL()); + return; + } + + // Here we parsed the URL correctly + this.resolveStartPromise( + `${url.protocol}//${url.host}${url.pathname}`, + `${url.searchParams.get('token')}`, + url.hostname + ); + } + } + + // tslint:disable-next-line:no-any + private extractConnectionInformation = (data: any) => { + this.output(data); + + const httpMatch = RegExpValues.HttpPattern.exec(data); + + if (httpMatch && this.notebookDir && this.startPromise && !this.startPromise.completed && this.getServerInfo) { + // .then so that we can keep from pushing aync up to the subscribed observable function + this.getServerInfo(this.cancelToken) + .then((serverInfos) => this.getJupyterURL(serverInfos, data)) + .catch((ex) => traceWarning('Failed to get server info', ex)); + } + + // Sometimes jupyter will return a 403 error. Not sure why. We used + // to fail on this, but it looks like jupyter works with this error in place. + }; + + private launchTimedOut = () => { + if (!this.startPromise.completed) { + this.rejectStartPromise(localize.DataScience.jupyterLaunchTimedOut()); + } + }; + + private resolveStartPromise = (baseUrl: string, token: string, hostName: string) => { + // tslint:disable-next-line: no-any + clearTimeout(this.launchTimeout as any); + if (!this.startPromise.rejected) { + const connection = this.createConnection(baseUrl, token, hostName, this.launchResult); + const origDispose = connection.dispose.bind(connection); + connection.dispose = () => { + // Stop listening when we disconnect + this.connectionDisposed = true; + return origDispose(); + }; + this.startPromise.resolve(connection); + } + }; + + // tslint:disable-next-line:no-any + private rejectStartPromise = (message: string) => { + // tslint:disable-next-line: no-any + clearTimeout(this.launchTimeout as any); + if (!this.startPromise.resolved) { + this.startPromise.reject( + Cancellation.isCanceled(this.cancelToken) + ? new CancellationError() + : new JupyterConnectError(message, this.stderr.join('\n')) + ); + } + }; +} + +// Represents an active connection to a running jupyter notebook +class JupyterConnection implements IConnection { + public readonly localLaunch: boolean = true; + public readonly type = 'jupyter'; + public valid: boolean = true; + public localProcExitCode: number | undefined; + private eventEmitter: EventEmitter = new EventEmitter(); + constructor( + public readonly baseUrl: string, + public readonly token: string, + public readonly hostName: string, + private readonly disposable: Disposable, + childProc: ChildProcess | undefined + ) { + // If the local process exits, set our exit code and fire our event + if (childProc) { + childProc.on('exit', (c) => { + // Our code expects the exit code to be of type `number` or `undefined`. + const code = typeof c === 'number' ? c : undefined; + this.valid = false; + this.localProcExitCode = code; + this.eventEmitter.fire(code); + }); + } + } + + public get displayName(): string { + return getJupyterConnectionDisplayName(this.token, this.baseUrl); + } + + public get disconnected(): Event { + return this.eventEmitter.event; + } + + public dispose() { + if (this.disposable) { + this.disposable.dispose(); + } + } +} + +export function getJupyterConnectionDisplayName(token: string, baseUrl: string): string { + const tokenString = token.length > 0 ? `?token=${token}` : ''; + return `${baseUrl}${tokenString}`; +} diff --git a/src/client/datascience/jupyter/jupyterDebugger.ts b/src/client/datascience/jupyter/jupyterDebugger.ts index 1b3461984789..538a3f0d6ac5 100644 --- a/src/client/datascience/jupyter/jupyterDebugger.ts +++ b/src/client/datascience/jupyter/jupyterDebugger.ts @@ -174,9 +174,11 @@ export class JupyterDebugger implements IJupyterDebugger, ICellHashListener { } // Connect local or remote based on what type of notebook we're talking to - const connectionInfo = notebook.server.getConnectionInfo(); + const connectionInfo = notebook.connection; if (connectionInfo && !connectionInfo.localLaunch) { - result = await this.connectToRemote(notebook, connectionInfo); + // Remote connections are always jupyter + const jupyterConnection = connectionInfo as IConnection; + result = await this.connectToRemote(notebook, jupyterConnection); } else { result = await this.connectToLocal(notebook); } @@ -232,7 +234,7 @@ export class JupyterDebugger implements IJupyterDebugger, ICellHashListener { // installed locally by the extension // Actually until this is resolved: https://github.com/microsoft/vscode-python/issues/7615, skip adding // this path. - const connectionInfo = notebook.server.getConnectionInfo(); + const connectionInfo = notebook.connection; if (connectionInfo && connectionInfo.localLaunch) { let localPath = await this.getDebuggerPath(notebook); if (this.platform.isWindows) { diff --git a/src/client/datascience/jupyter/jupyterNotebook.ts b/src/client/datascience/jupyter/jupyterNotebook.ts index b7032a9d8f0b..6d83d89d1a9b 100644 --- a/src/client/datascience/jupyter/jupyterNotebook.ts +++ b/src/client/datascience/jupyter/jupyterNotebook.ts @@ -32,9 +32,8 @@ import { IJupyterSession, INotebook, INotebookCompletion, + INotebookExecutionInfo, INotebookExecutionLogger, - INotebookServer, - INotebookServerLaunchInfo, InterruptResult, KernelSocketInformation } from '../types'; @@ -156,6 +155,7 @@ export class JupyterNotebookBase implements INotebook { private _identity: Uri; private _disposed: boolean = false; private _workingDirectory: string | undefined; + private _executionInfo: INotebookExecutionInfo; private onStatusChangedEvent: EventEmitter | undefined; public get onDisposed(): Event { return this.disposed.event; @@ -172,21 +172,16 @@ export class JupyterNotebookBase implements INotebook { private sessionStatusChanged: Disposable | undefined; private initializedMatplotlib = false; private ioPubListeners = new Set<(msg: KernelMessage.IIOPubMessage, requestId: string) => Promise>(); - private launchInfo: INotebookServerLaunchInfo; public get kernelSocket(): Observable { return this.session.kernelSocket; } - public get connection(): Readonly { - return this.launchInfo.connectionInfo; - } constructor( _liveShare: ILiveShareApi, // This is so the liveshare mixin works private session: IJupyterSession, private configService: IConfigurationService, private disposableRegistry: IDisposableRegistry, - private owner: INotebookServer, - _launchInfo: INotebookServerLaunchInfo, + executionInfo: INotebookExecutionInfo, private loggers: INotebookExecutionLogger[], resource: Resource, identity: Uri, @@ -207,10 +202,11 @@ export class JupyterNotebookBase implements INotebook { this._resource = resource; // Make a copy of the launch info so we can update it in this class - this.launchInfo = cloneDeep(_launchInfo); + this._executionInfo = cloneDeep(executionInfo); } - public get server(): INotebookServer { - return this.owner; + + public get connection() { + return this._executionInfo.connectionInfo; } public async dispose(): Promise { @@ -603,11 +599,11 @@ export class JupyterNotebookBase implements INotebook { } public getMatchingInterpreter(): PythonInterpreter | undefined { - return this.launchInfo.interpreter; + return this._executionInfo.interpreter; } public getKernelSpec(): IJupyterKernelSpec | LiveKernelModel | undefined { - return this.launchInfo.kernelSpec; + return this._executionInfo.kernelSpec; } public async setKernelSpec( @@ -625,20 +621,20 @@ export class JupyterNotebookBase implements INotebook { // Change our own kernel spec // Only after session was successfully created. - this.launchInfo.kernelSpec = spec; + this._executionInfo.kernelSpec = spec; // Rerun our initial setup await this.initialize(); } else { // Change our own kernel spec - this.launchInfo.kernelSpec = spec; + this._executionInfo.kernelSpec = spec; } this.kernelChanged.fire(spec); // If our new kernelspec has an interpreter, set that as our interpreter too if (interpreter) { - this.launchInfo.interpreter = interpreter; + this._executionInfo.interpreter = interpreter; } } @@ -902,16 +898,20 @@ export class JupyterNotebookBase implements INotebook { }; private async updateWorkingDirectoryAndPath(launchingFile?: string): Promise { - if (this.launchInfo && this.launchInfo.connectionInfo.localLaunch && !this._workingDirectory) { + if (this._executionInfo && this._executionInfo.connectionInfo.localLaunch && !this._workingDirectory) { // See what our working dir is supposed to be - const suggested = this.launchInfo.workingDir; + const suggested = this._executionInfo.workingDir; if (suggested && (await this.fs.directoryExists(suggested))) { // We should use the launch info directory. It trumps the possible dir this._workingDirectory = suggested; return this.changeDirectoryIfPossible(this._workingDirectory); } else if (launchingFile && (await this.fs.fileExists(launchingFile))) { // Combine the working directory with this file if possible. - this._workingDirectory = expandWorkingDir(this.launchInfo.workingDir, launchingFile, this.workspace); + this._workingDirectory = expandWorkingDir( + this._executionInfo.workingDir, + launchingFile, + this.workspace + ); if (this._workingDirectory) { return this.changeDirectoryIfPossible(this._workingDirectory); } @@ -922,8 +922,8 @@ export class JupyterNotebookBase implements INotebook { // Update both current working directory and sys.path with the desired directory private changeDirectoryIfPossible = async (directory: string): Promise => { if ( - this.launchInfo && - this.launchInfo.connectionInfo.localLaunch && + this._executionInfo && + this._executionInfo.connectionInfo.localLaunch && (await this.fs.directoryExists(directory)) ) { await this.executeSilently(CodeSnippits.UpdateCWDAndPath.format(directory)); @@ -994,11 +994,16 @@ export class JupyterNotebookBase implements INotebook { } private checkForExit(): Error | undefined { - if (this.launchInfo && this.launchInfo.connectionInfo && this.launchInfo.connectionInfo.localProcExitCode) { - // Not running, just exit - const exitCode = this.launchInfo.connectionInfo.localProcExitCode; - traceError(`Jupyter crashed with code ${exitCode}`); - return new Error(localize.DataScience.jupyterServerCrashed().format(exitCode.toString())); + if (this._executionInfo && this._executionInfo.connectionInfo && !this._executionInfo.connectionInfo.valid) { + if (this._executionInfo.connectionInfo.type === 'jupyter') { + const jupyterConnection = this._executionInfo.connectionInfo as IConnection; + // Not running, just exit + if (jupyterConnection.localProcExitCode) { + const exitCode = jupyterConnection.localProcExitCode; + traceError(`Jupyter crashed with code ${exitCode}`); + return new Error(localize.DataScience.jupyterServerCrashed().format(exitCode.toString())); + } + } } return undefined; @@ -1062,9 +1067,9 @@ export class JupyterNotebookBase implements INotebook { // Make sure our connection doesn't go down let exitHandlerDisposable: Disposable | undefined; - if (this.launchInfo && this.launchInfo.connectionInfo) { + if (this._executionInfo && this._executionInfo.connectionInfo) { // If the server crashes, cancel the current observable - exitHandlerDisposable = this.launchInfo.connectionInfo.disconnected((c) => { + exitHandlerDisposable = this._executionInfo.connectionInfo.disconnected((c) => { const str = c ? c.toString() : ''; // Only do an error if we're not disposed. If we're disposed we already shutdown. if (!this._disposed) { diff --git a/src/client/datascience/jupyter/jupyterUtils.ts b/src/client/datascience/jupyter/jupyterUtils.ts index 0a54621e123d..ed8a157aec06 100644 --- a/src/client/datascience/jupyter/jupyterUtils.ts +++ b/src/client/datascience/jupyter/jupyterUtils.ts @@ -11,6 +11,7 @@ import { IDataScienceSettings } from '../../common/types'; import { noop } from '../../common/utils/misc'; import { SystemVariables } from '../../common/variables/systemVariables'; import { Identifiers } from '../constants'; +import { getJupyterConnectionDisplayName } from '../jupyter/jupyterConnection'; import { IConnection } from '../types'; // tslint:disable-next-line:no-require-imports no-var-requires @@ -42,13 +43,19 @@ export function createRemoteConnectionInfo(uri: string, settings: IDataScienceSe ? settings.allowUnauthorizedRemoteConnection : false; + const baseUrl = `${url.protocol}//${url.host}${url.pathname}`; + const token = `${url.searchParams.get('token')}`; + return { + type: 'jupyter', allowUnauthorized, - baseUrl: `${url.protocol}//${url.host}${url.pathname}`, - token: `${url.searchParams.get('token')}`, + baseUrl, + token, hostName: url.hostname, localLaunch: false, localProcExitCode: undefined, + valid: true, + displayName: getJupyterConnectionDisplayName(token, baseUrl), disconnected: (_l) => { return { dispose: noop }; }, diff --git a/src/client/datascience/jupyter/kernels/kernelSwitcher.ts b/src/client/datascience/jupyter/kernels/kernelSwitcher.ts index 0d5e794caa54..46e1d4ecb8f3 100644 --- a/src/client/datascience/jupyter/kernels/kernelSwitcher.ts +++ b/src/client/datascience/jupyter/kernels/kernelSwitcher.ts @@ -52,16 +52,18 @@ export class KernelSwitcher { const settings = this.configService.getSettings(notebook.resource); const isLocalConnection = - notebook.server.getConnectionInfo()?.localLaunch ?? + notebook.connection?.localLaunch ?? settings.datascience.jupyterServerURI.toLowerCase() === Settings.JupyterServerLocalLaunch; if (isLocalConnection) { kernel = await this.selectLocalJupyterKernel(notebook.resource, notebook?.getKernelSpec()); } else if (notebook) { - const connInfo = notebook.server.getConnectionInfo(); + const connInfo = notebook.connection; const currentKernel = notebook.getKernelSpec(); if (connInfo) { - kernel = await this.selectRemoteJupyterKernel(notebook.resource, connInfo, currentKernel); + // Remote connection is always jupyter connection + const jupyterConnInfo = connInfo as IConnection; + kernel = await this.selectRemoteJupyterKernel(notebook.resource, jupyterConnInfo, currentKernel); } } return kernel; @@ -86,7 +88,7 @@ export class KernelSwitcher { private async switchKernelWithRetry(notebook: INotebook, kernel: KernelSpecInterpreter): Promise { const settings = this.configService.getSettings(notebook.resource); const isLocalConnection = - notebook.server.getConnectionInfo()?.localLaunch ?? + notebook.connection?.localLaunch ?? settings.datascience.jupyterServerURI.toLowerCase() === Settings.JupyterServerLocalLaunch; if (!isLocalConnection) { await this.switchToKernel(notebook, kernel); diff --git a/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts b/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts index 66a5f7d37f34..4950c265f750 100644 --- a/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts +++ b/src/client/datascience/jupyter/liveshare/guestJupyterNotebook.ts @@ -19,12 +19,12 @@ import { PythonInterpreter } from '../../../interpreter/contracts'; import { LiveShare, LiveShareCommands } from '../../constants'; import { ICell, - IConnection, IJupyterKernelSpec, INotebook, INotebookCompletion, + INotebookExecutionInfo, INotebookExecutionLogger, - INotebookServer, + INotebookProviderConnection, InterruptResult, KernelSocketInformation } from '../../types'; @@ -44,10 +44,6 @@ export class GuestJupyterNotebook return this._jupyterLab; } - public get connection(): IConnection { - throw new Error('Not Implemented'); - } - public get identity(): Uri { return this._identity; } @@ -56,8 +52,8 @@ export class GuestJupyterNotebook return this._resource; } - public get server(): INotebookServer { - return this._owner; + public get connection(): INotebookProviderConnection | undefined { + return this._executionInfo?.connectionInfo; } public kernelSocket = new Observable(); @@ -87,7 +83,7 @@ export class GuestJupyterNotebook private configService: IConfigurationService, private _resource: Resource, private _identity: Uri, - private _owner: INotebookServer, + private _executionInfo: INotebookExecutionInfo | undefined, private startTime: number ) { super(liveShare); diff --git a/src/client/datascience/jupyter/liveshare/guestJupyterServer.ts b/src/client/datascience/jupyter/liveshare/guestJupyterServer.ts index 8cdfd3d706a0..f183a83e3286 100644 --- a/src/client/datascience/jupyter/liveshare/guestJupyterServer.ts +++ b/src/client/datascience/jupyter/liveshare/guestJupyterServer.ts @@ -77,7 +77,7 @@ export class GuestJupyterServer this.configService, resource, identity, - this, + this.launchInfo, this.dataScience.activationStartTime ); deferred.resolve(result); diff --git a/src/client/datascience/jupyter/liveshare/hostJupyterExecution.ts b/src/client/datascience/jupyter/liveshare/hostJupyterExecution.ts index d55c95599eaf..5948b5079fd4 100644 --- a/src/client/datascience/jupyter/liveshare/hostJupyterExecution.ts +++ b/src/client/datascience/jupyter/liveshare/hostJupyterExecution.ts @@ -1,168 +1,172 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. -'use strict'; -import '../../../common/extensions'; - -import { CancellationToken } from 'vscode'; -import * as vsls from 'vsls/vscode'; - -import { IApplicationShell, ILiveShareApi, IWorkspaceService } from '../../../common/application/types'; -import { IFileSystem } from '../../../common/platform/types'; -import { - IAsyncDisposableRegistry, - IConfigurationService, - IDisposableRegistry, - IOutputChannel -} from '../../../common/types'; -import { noop } from '../../../common/utils/misc'; -import { IInterpreterService } from '../../../interpreter/contracts'; -import { IServiceContainer } from '../../../ioc/types'; -import { LiveShare, LiveShareCommands } from '../../constants'; -import { IConnection, IJupyterExecution, INotebookServer, INotebookServerOptions } from '../../types'; -import { JupyterExecutionBase } from '../jupyterExecution'; -import { KernelSelector } from '../kernels/kernelSelector'; -import { NotebookStarter } from '../notebookStarter'; -import { LiveShareParticipantHost } from './liveShareParticipantMixin'; -import { IRoleBasedObject } from './roleBasedFactory'; -import { ServerCache } from './serverCache'; - -// tslint:disable:no-any - -// This class is really just a wrapper around a jupyter execution that also provides a shared live share service -export class HostJupyterExecution - extends LiveShareParticipantHost(JupyterExecutionBase, LiveShare.JupyterExecutionService) - implements IRoleBasedObject, IJupyterExecution { - private serverCache: ServerCache; - constructor( - liveShare: ILiveShareApi, - interpreterService: IInterpreterService, - disposableRegistry: IDisposableRegistry, - asyncRegistry: IAsyncDisposableRegistry, - fileSys: IFileSystem, - workspace: IWorkspaceService, - configService: IConfigurationService, - kernelSelector: KernelSelector, - notebookStarter: NotebookStarter, - appShell: IApplicationShell, - jupyterOutputChannel: IOutputChannel, - serviceContainer: IServiceContainer - ) { - super( - liveShare, - interpreterService, - disposableRegistry, - workspace, - configService, - kernelSelector, - notebookStarter, - appShell, - jupyterOutputChannel, - serviceContainer - ); - this.serverCache = new ServerCache(configService, workspace, fileSys); - asyncRegistry.push(this); - } - - public async dispose(): Promise { - await super.dispose(); - const api = await this.api; - await this.onDetach(api); - - // Cleanup on dispose. We are going away permanently - if (this.serverCache) { - await this.serverCache.dispose(); - } - } - - public async hostConnectToNotebookServer( - options?: INotebookServerOptions, - cancelToken?: CancellationToken - ): Promise { - return super.connectToNotebookServer(await this.serverCache.generateDefaultOptions(options), cancelToken); - } - - public async connectToNotebookServer( - options?: INotebookServerOptions, - cancelToken?: CancellationToken - ): Promise { - return this.serverCache.getOrCreate(this.hostConnectToNotebookServer.bind(this), options, cancelToken); - } - - public async onAttach(api: vsls.LiveShare | null): Promise { - await super.onAttach(api); - - if (api) { - const service = await this.waitForService(); - - // Register handlers for all of the supported remote calls - if (service) { - service.onRequest(LiveShareCommands.isNotebookSupported, this.onRemoteIsNotebookSupported); - service.onRequest(LiveShareCommands.isImportSupported, this.onRemoteIsImportSupported); - service.onRequest(LiveShareCommands.connectToNotebookServer, this.onRemoteConnectToNotebookServer); - service.onRequest(LiveShareCommands.getUsableJupyterPython, this.onRemoteGetUsableJupyterPython); - } - } - } - - public async onDetach(api: vsls.LiveShare | null): Promise { - await super.onDetach(api); - - // clear our cached servers if our role is no longer host or none - const newRole = - api === null || (api.session && api.session.role !== vsls.Role.Guest) ? vsls.Role.Host : vsls.Role.Guest; - if (newRole !== vsls.Role.Host) { - await this.serverCache.dispose(); - } - } - - public getServer(options?: INotebookServerOptions): Promise { - // See if we have this server or not. - return this.serverCache.get(options); - } - - private onRemoteIsNotebookSupported = (_args: any[], cancellation: CancellationToken): Promise => { - // Just call local - return this.isNotebookSupported(cancellation); - }; - - private onRemoteIsImportSupported = (_args: any[], cancellation: CancellationToken): Promise => { - // Just call local - return this.isImportSupported(cancellation); - }; - - private onRemoteConnectToNotebookServer = async ( - args: any[], - cancellation: CancellationToken - ): Promise => { - // Connect to the local server. THe local server should have started the port forwarding already - const localServer = await this.connectToNotebookServer( - args[0] as INotebookServerOptions | undefined, - cancellation - ); - - // Extract the URI and token for the other side - if (localServer) { - // The other side should be using 'localhost' for anything it's port forwarding. That should just remap - // on the guest side. However we need to eliminate the dispose method. Methods are not serializable - const connectionInfo = localServer.getConnectionInfo(); - if (connectionInfo) { - return { - baseUrl: connectionInfo.baseUrl, - token: connectionInfo.token, - hostName: connectionInfo.hostName, - localLaunch: false, - localProcExitCode: undefined, - disconnected: (_l) => { - return { dispose: noop }; - }, - dispose: noop - }; - } - } - }; - - private onRemoteGetUsableJupyterPython = (_args: any[], cancellation: CancellationToken): Promise => { - // Just call local - return this.getUsableJupyterPython(cancellation); - }; -} +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +'use strict'; +import '../../../common/extensions'; + +import { CancellationToken } from 'vscode'; +import * as vsls from 'vsls/vscode'; + +import { IApplicationShell, ILiveShareApi, IWorkspaceService } from '../../../common/application/types'; +import { IFileSystem } from '../../../common/platform/types'; +import { + IAsyncDisposableRegistry, + IConfigurationService, + IDisposableRegistry, + IOutputChannel +} from '../../../common/types'; +import { noop } from '../../../common/utils/misc'; +import { IInterpreterService } from '../../../interpreter/contracts'; +import { IServiceContainer } from '../../../ioc/types'; +import { LiveShare, LiveShareCommands } from '../../constants'; +import { IConnection, IJupyterExecution, INotebookServer, INotebookServerOptions } from '../../types'; +import { getJupyterConnectionDisplayName } from '../jupyterConnection'; +import { JupyterExecutionBase } from '../jupyterExecution'; +import { KernelSelector } from '../kernels/kernelSelector'; +import { NotebookStarter } from '../notebookStarter'; +import { LiveShareParticipantHost } from './liveShareParticipantMixin'; +import { IRoleBasedObject } from './roleBasedFactory'; +import { ServerCache } from './serverCache'; + +// tslint:disable:no-any + +// This class is really just a wrapper around a jupyter execution that also provides a shared live share service +export class HostJupyterExecution + extends LiveShareParticipantHost(JupyterExecutionBase, LiveShare.JupyterExecutionService) + implements IRoleBasedObject, IJupyterExecution { + private serverCache: ServerCache; + constructor( + liveShare: ILiveShareApi, + interpreterService: IInterpreterService, + disposableRegistry: IDisposableRegistry, + asyncRegistry: IAsyncDisposableRegistry, + fileSys: IFileSystem, + workspace: IWorkspaceService, + configService: IConfigurationService, + kernelSelector: KernelSelector, + notebookStarter: NotebookStarter, + appShell: IApplicationShell, + jupyterOutputChannel: IOutputChannel, + serviceContainer: IServiceContainer + ) { + super( + liveShare, + interpreterService, + disposableRegistry, + workspace, + configService, + kernelSelector, + notebookStarter, + appShell, + jupyterOutputChannel, + serviceContainer + ); + this.serverCache = new ServerCache(configService, workspace, fileSys); + asyncRegistry.push(this); + } + + public async dispose(): Promise { + await super.dispose(); + const api = await this.api; + await this.onDetach(api); + + // Cleanup on dispose. We are going away permanently + if (this.serverCache) { + await this.serverCache.dispose(); + } + } + + public async hostConnectToNotebookServer( + options?: INotebookServerOptions, + cancelToken?: CancellationToken + ): Promise { + return super.connectToNotebookServer(await this.serverCache.generateDefaultOptions(options), cancelToken); + } + + public async connectToNotebookServer( + options?: INotebookServerOptions, + cancelToken?: CancellationToken + ): Promise { + return this.serverCache.getOrCreate(this.hostConnectToNotebookServer.bind(this), options, cancelToken); + } + + public async onAttach(api: vsls.LiveShare | null): Promise { + await super.onAttach(api); + + if (api) { + const service = await this.waitForService(); + + // Register handlers for all of the supported remote calls + if (service) { + service.onRequest(LiveShareCommands.isNotebookSupported, this.onRemoteIsNotebookSupported); + service.onRequest(LiveShareCommands.isImportSupported, this.onRemoteIsImportSupported); + service.onRequest(LiveShareCommands.connectToNotebookServer, this.onRemoteConnectToNotebookServer); + service.onRequest(LiveShareCommands.getUsableJupyterPython, this.onRemoteGetUsableJupyterPython); + } + } + } + + public async onDetach(api: vsls.LiveShare | null): Promise { + await super.onDetach(api); + + // clear our cached servers if our role is no longer host or none + const newRole = + api === null || (api.session && api.session.role !== vsls.Role.Guest) ? vsls.Role.Host : vsls.Role.Guest; + if (newRole !== vsls.Role.Host) { + await this.serverCache.dispose(); + } + } + + public getServer(options?: INotebookServerOptions): Promise { + // See if we have this server or not. + return this.serverCache.get(options); + } + + private onRemoteIsNotebookSupported = (_args: any[], cancellation: CancellationToken): Promise => { + // Just call local + return this.isNotebookSupported(cancellation); + }; + + private onRemoteIsImportSupported = (_args: any[], cancellation: CancellationToken): Promise => { + // Just call local + return this.isImportSupported(cancellation); + }; + + private onRemoteConnectToNotebookServer = async ( + args: any[], + cancellation: CancellationToken + ): Promise => { + // Connect to the local server. THe local server should have started the port forwarding already + const localServer = await this.connectToNotebookServer( + args[0] as INotebookServerOptions | undefined, + cancellation + ); + + // Extract the URI and token for the other side + if (localServer) { + // The other side should be using 'localhost' for anything it's port forwarding. That should just remap + // on the guest side. However we need to eliminate the dispose method. Methods are not serializable + const connectionInfo = localServer.getConnectionInfo(); + if (connectionInfo) { + return { + type: 'jupyter', + baseUrl: connectionInfo.baseUrl, + token: connectionInfo.token, + hostName: connectionInfo.hostName, + localLaunch: false, + localProcExitCode: undefined, + valid: true, + displayName: getJupyterConnectionDisplayName(connectionInfo.token, connectionInfo.baseUrl), + disconnected: (_l) => { + return { dispose: noop }; + }, + dispose: noop + }; + } + } + }; + + private onRemoteGetUsableJupyterPython = (_args: any[], cancellation: CancellationToken): Promise => { + // Just call local + return this.getUsableJupyterPython(cancellation); + }; +} diff --git a/src/client/datascience/jupyter/liveshare/hostJupyterNotebook.ts b/src/client/datascience/jupyter/liveshare/hostJupyterNotebook.ts index 1a7c6f429e4f..40620f640c58 100644 --- a/src/client/datascience/jupyter/liveshare/hostJupyterNotebook.ts +++ b/src/client/datascience/jupyter/liveshare/hostJupyterNotebook.ts @@ -17,9 +17,8 @@ import { ICell, IJupyterSession, INotebook, + INotebookExecutionInfo, INotebookExecutionLogger, - INotebookServer, - INotebookServerLaunchInfo, InterruptResult } from '../../types'; import { JupyterNotebookBase } from '../jupyterNotebook'; @@ -46,8 +45,7 @@ export class HostJupyterNotebook session: IJupyterSession, configService: IConfigurationService, disposableRegistry: IDisposableRegistry, - owner: INotebookServer, - launchInfo: INotebookServerLaunchInfo, + executionInfo: INotebookExecutionInfo, loggers: INotebookExecutionLogger[], resource: Resource, identity: vscode.Uri, @@ -61,8 +59,7 @@ export class HostJupyterNotebook session, configService, disposableRegistry, - owner, - launchInfo, + executionInfo, loggers, resource, identity, diff --git a/src/client/datascience/jupyter/liveshare/hostJupyterServer.ts b/src/client/datascience/jupyter/liveshare/hostJupyterServer.ts index 968fc915dcc4..841243c13a00 100644 --- a/src/client/datascience/jupyter/liveshare/hostJupyterServer.ts +++ b/src/client/datascience/jupyter/liveshare/hostJupyterServer.ts @@ -228,7 +228,6 @@ export class HostJupyterServer extends LiveShareParticipantHost(JupyterServerBas session, configService, disposableRegistry, - this, info, serviceContainer.getAll(INotebookExecutionLogger), resource, diff --git a/src/client/datascience/jupyter/serverPreload.ts b/src/client/datascience/jupyter/serverPreload.ts index fd1ef00e9446..347c3af95a9f 100644 --- a/src/client/datascience/jupyter/serverPreload.ts +++ b/src/client/datascience/jupyter/serverPreload.ts @@ -53,20 +53,20 @@ export class ServerPreload implements IExtensionSingleActivationService { try { traceInfo(`Attempting to start a server because of preload conditions ...`); - // May already have this server started. - let server = await this.notebookProvider.getOrCreateServer({ getOnly: true, disableUI: true }); + // Check if we are already connected + let providerConnection = await this.notebookProvider.connect({ getOnly: true, disableUI: true }); // If it didn't start, attempt for local and if allowed. - if (!server && !this.configService.getSettings(undefined).datascience.disableJupyterAutoStart) { + if (!providerConnection && !this.configService.getSettings(undefined).datascience.disableJupyterAutoStart) { // Local case, try creating one - server = await this.notebookProvider.getOrCreateServer({ + providerConnection = await this.notebookProvider.connect({ getOnly: false, disableUI: true, localOnly: true }); } - if (server) { + if (providerConnection) { // Update our date in the storage that indicates it was succesful this.mementoStorage.update(LastServerActiveTimeKey, Date.now()); } diff --git a/src/client/datascience/serviceRegistry.ts b/src/client/datascience/serviceRegistry.ts index 71e15ce806af..96b847d6b4d1 100644 --- a/src/client/datascience/serviceRegistry.ts +++ b/src/client/datascience/serviceRegistry.ts @@ -30,6 +30,7 @@ import { DebugListener } from './interactive-common/debugListener'; import { IntellisenseProvider } from './interactive-common/intellisense/intellisenseProvider'; import { LinkProvider } from './interactive-common/linkProvider'; import { NotebookProvider } from './interactive-common/notebookProvider'; +import { NotebookServerProvider } from './interactive-common/notebookServerProvider'; import { ShowPlotListener } from './interactive-common/showPlotListener'; import { AutoSaveService } from './interactive-ipynb/autoSaveService'; import { NativeEditor } from './interactive-ipynb/nativeEditor'; @@ -117,6 +118,7 @@ import { INotebookImporter, INotebookProvider, INotebookServer, + INotebookServerProvider, INotebookStorage, IPlotViewer, IPlotViewerProvider, @@ -200,6 +202,7 @@ export function registerTypes(serviceManager: IServiceManager) { serviceManager.addSingleton(ProgressReporter, ProgressReporter); serviceManager.addSingleton(NativeEditorSynchronizer, NativeEditorSynchronizer); serviceManager.addSingleton(INotebookProvider, NotebookProvider); + serviceManager.addSingleton(INotebookServerProvider, NotebookServerProvider); serviceManager.add(IJMPConnection, EnchannelJMPConnection); serviceManager.addSingleton(IPyWidgetMessageDispatcherFactory, IPyWidgetMessageDispatcherFactory); diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index 2ae2275cbfcc..e419885d18c7 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -48,14 +48,29 @@ export interface IDataScienceCommandListener { register(commandManager: ICommandManager): void; } -// Connection information for talking to a jupyter notebook process -export interface IConnection extends Disposable { +// Connection information for talking to a generic notebook provider +export interface INotebookProviderConnection extends Disposable { + // What type of notebook provider are we connected to + readonly type: 'raw' | 'jupyter'; + // Was this connection launched locally or not + readonly localLaunch: boolean; + // Is the connection still valid + readonly valid: boolean; + // Display name + readonly displayName: string; + // Called if whatever provides the notebook is disconnected + disconnected: Event; +} + +// Connection information for talking to a raw ZMQ provider +export interface IRawConnection extends INotebookProviderConnection {} + +// Connection information for talking to a jupyter server process +export interface IConnection extends INotebookProviderConnection { readonly baseUrl: string; readonly token: string; readonly hostName: string; - readonly localLaunch: boolean; localProcExitCode: number | undefined; - disconnected: Event; allowUnauthorized?: boolean; } @@ -65,6 +80,23 @@ export enum InterruptResult { Restarted = 2 } +// Information used to execute a notebook +export interface INotebookExecutionInfo { + // Connection to what has provided our notebook, such as a jupyter + // server or a raw ZMQ kernel + connectionInfo: INotebookProviderConnection; + /** + * The python interpreter associated with the kernel. + */ + interpreter: PythonInterpreter | undefined; + uri: string | undefined; // Different from the connectionInfo as this is the setting used, not the result + kernelSpec: IJupyterKernelSpec | undefined | LiveKernelModel; + workingDir: string | undefined; + purpose: string | undefined; // Purpose this server is for +} + +// Information used to launch a jupyter notebook server + // Information used to launch a notebook server export interface INotebookServerLaunchInfo { connectionInfo: IConnection; @@ -109,10 +141,9 @@ export interface INotebookServer extends IAsyncDisposable { export interface INotebook extends IAsyncDisposable { readonly resource: Resource; - readonly connection: Readonly; + readonly connection: INotebookProviderConnection | undefined; kernelSocket: Observable; readonly identity: Uri; - readonly server: INotebookServer; readonly status: ServerStatus; onSessionStatusChanged: Event; onDisposed: Event; @@ -172,6 +203,13 @@ export interface INotebook extends IAsyncDisposable { removeMessageHook(msgId: string, hook: (msg: KernelMessage.IIOPubMessage) => boolean | PromiseLike): void; } +// Options for connecting to a notebook provider +export type ConnectNotebookProviderOptions = { + getOnly?: boolean; + disableUI?: boolean; + localOnly?: boolean; +}; + export interface INotebookServerOptions { uri?: string; usingDarkTheme?: boolean; @@ -955,7 +993,19 @@ export interface INotebookProvider { * Gets or creates a notebook, and manages the lifetime of notebooks. */ getOrCreateNotebook(options: GetNotebookOptions): Promise; + /** + * Connect to a notebook provider to prepare its connection and to get connection information + */ + connect(options: ConnectNotebookProviderOptions): Promise; + + /** + * Disconnect from a notebook provider connection + */ + disconnect(options: ConnectNotebookProviderOptions): Promise; +} +export const INotebookServerProvider = Symbol('INotebookServerProvider'); +export interface INotebookServerProvider { /** * Gets the server used for starting notebooks */ diff --git a/src/test/datascience/dataScienceIocContainer.ts b/src/test/datascience/dataScienceIocContainer.ts index 1b8d83ba8d31..0520dc4ed33b 100644 --- a/src/test/datascience/dataScienceIocContainer.ts +++ b/src/test/datascience/dataScienceIocContainer.ts @@ -183,6 +183,7 @@ import { GatherListener } from '../../client/datascience/gather/gatherListener'; import { GatherLogger } from '../../client/datascience/gather/gatherLogger'; import { IntellisenseProvider } from '../../client/datascience/interactive-common/intellisense/intellisenseProvider'; import { NotebookProvider } from '../../client/datascience/interactive-common/notebookProvider'; +import { NotebookServerProvider } from '../../client/datascience/interactive-common/notebookServerProvider'; import { AutoSaveService } from '../../client/datascience/interactive-ipynb/autoSaveService'; import { NativeEditor } from '../../client/datascience/interactive-ipynb/nativeEditor'; import { NativeEditorCommandListener } from '../../client/datascience/interactive-ipynb/nativeEditorCommandListener'; @@ -263,6 +264,7 @@ import { INotebookImporter, INotebookProvider, INotebookServer, + INotebookServerProvider, INotebookStorage, IPlotViewer, IPlotViewerProvider, @@ -701,6 +703,7 @@ export class DataScienceIocContainer extends UnitTestIocContainer { } this.serviceManager.addSingleton(INotebookProvider, NotebookProvider); + this.serviceManager.addSingleton(INotebookServerProvider, NotebookServerProvider); this.serviceManager.add(IInteractiveWindowListener, IntellisenseProvider); this.serviceManager.add(IInteractiveWindowListener, AutoSaveService); diff --git a/src/test/datascience/interactive-common/notebookProvider.unit.test.ts b/src/test/datascience/interactive-common/notebookProvider.unit.test.ts new file mode 100644 index 000000000000..a40070012a52 --- /dev/null +++ b/src/test/datascience/interactive-common/notebookProvider.unit.test.ts @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +import { expect } from 'chai'; +import { anything, instance, mock, when } from 'ts-mockito'; +import * as typemoq from 'typemoq'; +import * as vscode from 'vscode'; +import { IFileSystem } from '../../../client/common/platform/types'; +import { IDisposableRegistry } from '../../../client/common/types'; +import { NotebookProvider } from '../../../client/datascience/interactive-common/notebookProvider'; +import { + IInteractiveWindowProvider, + INotebook, + INotebookEditorProvider, + INotebookServer, + INotebookServerProvider +} from '../../../client/datascience/types'; + +function Uri(filename: string): vscode.Uri { + return vscode.Uri.file(filename); +} + +// tslint:disable:no-any +function createTypeMoq(tag: string): typemoq.IMock { + // Use typemoqs for those things that are resolved as promises. mockito doesn't allow nesting of mocks. ES6 Proxy class + // is the problem. We still need to make it thenable though. See this issue: https://github.com/florinn/typemoq/issues/67 + const result = typemoq.Mock.ofType(); + (result as any).tag = tag; + result.setup((x: any) => x.then).returns(() => undefined); + return result; +} + +// tslint:disable: max-func-body-length +suite('Data Science - NotebookProvider', () => { + let notebookProvider: NotebookProvider; + let fileSystem: IFileSystem; + let notebookEditorProvider: INotebookEditorProvider; + let interactiveWindowProvider: IInteractiveWindowProvider; + let disposableRegistry: IDisposableRegistry; + let notebookServerProvider: INotebookServerProvider; + + setup(() => { + fileSystem = mock(); + notebookEditorProvider = mock(); + interactiveWindowProvider = mock(); + disposableRegistry = mock(); + notebookServerProvider = mock(); + notebookProvider = new NotebookProvider( + instance(fileSystem), + instance(notebookEditorProvider), + instance(interactiveWindowProvider), + instance(disposableRegistry), + instance(notebookServerProvider) + ); + }); + + test('NotebookProvider getOrCreateNotebook no server', async () => { + when(notebookServerProvider.getOrCreateServer(anything())).thenResolve(undefined); + + const notebook = await notebookProvider.getOrCreateNotebook({ identity: Uri('C:\\\\foo.py') }); + expect(notebook).to.equal(undefined, 'No server should return no notebook'); + }); + + test('NotebookProvider getOrCreateNotebook server has notebook already', async () => { + const notebookServer = createTypeMoq('jupyter server'); + const notebookMock = createTypeMoq('jupyter notebook'); + notebookServer + .setup(s => s.getNotebook(typemoq.It.isAny())) + .returns(() => Promise.resolve(notebookMock.object)); + when(notebookServerProvider.getOrCreateServer(anything())).thenResolve(notebookServer.object); + + const notebook = await notebookProvider.getOrCreateNotebook({ identity: Uri('C:\\\\foo.py') }); + expect(notebook).to.not.equal(undefined, 'Server should return a notebook'); + }); + + test('NotebookProvider getOrCreateNotebook server does not have notebook already', async () => { + const notebookServer = createTypeMoq('jupyter server'); + const notebookMock = createTypeMoq('jupyter notebook'); + // Get notebook undefined, but create notebook set + notebookServer.setup(s => s.getNotebook(typemoq.It.isAny())).returns(() => Promise.resolve(undefined)); + notebookServer + .setup(s => s.createNotebook(typemoq.It.isAny(), typemoq.It.isAny(), typemoq.It.isAny())) + .returns(() => Promise.resolve(notebookMock.object)); + when(notebookServerProvider.getOrCreateServer(anything())).thenResolve(notebookServer.object); + + const notebook = await notebookProvider.getOrCreateNotebook({ identity: Uri('C:\\\\foo.py') }); + expect(notebook).to.not.equal(undefined, 'Server should return a notebook'); + }); + + test('NotebookProvider getOrCreateNotebook getOnly server does not have notebook already', async () => { + const notebookServer = createTypeMoq('jupyter server'); + const notebookMock = createTypeMoq('jupyter notebook'); + // Get notebook undefined, but create notebook set + notebookServer.setup(s => s.getNotebook(typemoq.It.isAny())).returns(() => Promise.resolve(undefined)); + notebookServer + .setup(s => s.createNotebook(typemoq.It.isAny(), typemoq.It.isAny(), typemoq.It.isAny())) + .returns(() => Promise.resolve(notebookMock.object)); + when(notebookServerProvider.getOrCreateServer(anything())).thenResolve(notebookServer.object); + + const notebook = await notebookProvider.getOrCreateNotebook({ identity: Uri('C:\\\\foo.py') }); + expect(notebook).to.not.equal(undefined, 'Server should return a notebook'); + }); + + test('NotebookProvider getOrCreateNotebook second request should return the notebook already cached', async () => { + const notebookServer = createTypeMoq('jupyter server'); + const notebookMock = createTypeMoq('jupyter notebook'); + // Get notebook undefined, but create notebook set + notebookServer.setup(s => s.getNotebook(typemoq.It.isAny())).returns(() => Promise.resolve(undefined)); + notebookServer + .setup(s => s.createNotebook(typemoq.It.isAny(), typemoq.It.isAny(), typemoq.It.isAny())) + .returns(() => Promise.resolve(notebookMock.object)); + when(notebookServerProvider.getOrCreateServer(anything())).thenResolve(notebookServer.object); + + const notebook = await notebookProvider.getOrCreateNotebook({ identity: Uri('C:\\\\foo.py') }); + expect(notebook).to.not.equal(undefined, 'Server should return a notebook'); + + const notebook2 = await notebookProvider.getOrCreateNotebook({ identity: Uri('C:\\\\foo.py') }); + expect(notebook2).to.equal(notebook); + + // Only one create call + notebookServer.verify( + s => s.createNotebook(typemoq.It.isAny(), typemoq.It.isAny(), typemoq.It.isAny()), + typemoq.Times.once() + ); + }); +}); diff --git a/src/test/datascience/interactive-common/notebookServerProvider.unit.test.ts b/src/test/datascience/interactive-common/notebookServerProvider.unit.test.ts new file mode 100644 index 000000000000..5da6eecbd31b --- /dev/null +++ b/src/test/datascience/interactive-common/notebookServerProvider.unit.test.ts @@ -0,0 +1,95 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +import { expect } from 'chai'; +import { SemVer } from 'semver'; +import { anything, instance, mock, verify, when } from 'ts-mockito'; +import * as typemoq from 'typemoq'; +import { IApplicationShell } from '../../../client/common/application/types'; +import { IConfigurationService, IDataScienceSettings, IPythonSettings } from '../../../client/common/types'; +import { Architecture } from '../../../client/common/utils/platform'; +import { NotebookServerProvider } from '../../../client/datascience/interactive-common/notebookServerProvider'; +import { ProgressReporter } from '../../../client/datascience/progress/progressReporter'; +import { IJupyterExecution, INotebookServer } from '../../../client/datascience/types'; +import { IInterpreterService, InterpreterType, PythonInterpreter } from '../../../client/interpreter/contracts'; + +// tslint:disable:no-any +function createTypeMoq(tag: string): typemoq.IMock { + // Use typemoqs for those things that are resolved as promises. mockito doesn't allow nesting of mocks. ES6 Proxy class + // is the problem. We still need to make it thenable though. See this issue: https://github.com/florinn/typemoq/issues/67 + const result = typemoq.Mock.ofType(); + (result as any).tag = tag; + result.setup((x: any) => x.then).returns(() => undefined); + return result; +} + +// tslint:disable: max-func-body-length +suite('Data Science - NotebookServerProvider', () => { + let serverProvider: NotebookServerProvider; + let progressReporter: ProgressReporter; + let configurationService: IConfigurationService; + let jupyterExecution: IJupyterExecution; + let applicationShell: IApplicationShell; + let interpreterService: IInterpreterService; + let pythonSettings: IPythonSettings; + let dataScienceSettings: IDataScienceSettings; + const workingPython: PythonInterpreter = { + path: '/foo/bar/python.exe', + version: new SemVer('3.6.6-final'), + sysVersion: '1.0.0.0', + sysPrefix: 'Python', + type: InterpreterType.Unknown, + architecture: Architecture.x64 + }; + + setup(() => { + progressReporter = mock(ProgressReporter); + configurationService = mock(); + jupyterExecution = mock(); + applicationShell = mock(); + interpreterService = mock(); + + // Set up our settings + pythonSettings = mock(); + dataScienceSettings = mock(); + when(pythonSettings.datascience).thenReturn(instance(dataScienceSettings)); + when(dataScienceSettings.jupyterServerURI).thenReturn('local'); + when(dataScienceSettings.useDefaultConfigForJupyter).thenReturn(true); + when(configurationService.getSettings(anything())).thenReturn(instance(pythonSettings)); + + // Create the server provider + serverProvider = new NotebookServerProvider( + instance(progressReporter), + instance(configurationService), + instance(jupyterExecution), + instance(applicationShell), + instance(interpreterService) + ); + }); + + test('NotebookServerProvider - Get Only - no server', async () => { + when(jupyterExecution.getServer(anything())).thenResolve(undefined); + + const server = await serverProvider.getOrCreateServer({ getOnly: true }); + expect(server).to.equal(undefined, 'Server expected to be undefined'); + verify(jupyterExecution.getServer(anything())).once(); + }); + + test('NotebookServerProvider - Get Only - server', async () => { + const notebookServer = mock(); + when(jupyterExecution.getServer(anything())).thenResolve(instance(notebookServer)); + + const server = serverProvider.getOrCreateServer({ getOnly: true }); + expect(server).to.not.equal(undefined, 'Server expected to be defined'); + verify(jupyterExecution.getServer(anything())).once(); + }); + + test('NotebookServerProvider - Get Or Create', async () => { + when(jupyterExecution.getUsableJupyterPython()).thenResolve(workingPython); + const notebookServer = createTypeMoq('jupyter server'); + when(jupyterExecution.connectToNotebookServer(anything(), anything())).thenResolve(notebookServer.object); + + // Disable UI just lets us skip mocking the progress reporter + const server = await serverProvider.getOrCreateServer({ getOnly: false, disableUI: true }); + expect(server).to.not.equal(undefined, 'Server expected to be defined'); + }); +}); diff --git a/src/test/datascience/ipywidgets/cdnWidgetScriptSourceProvider.unit.test.ts b/src/test/datascience/ipywidgets/cdnWidgetScriptSourceProvider.unit.test.ts index 22f3faae9788..9a21e8feefb7 100644 --- a/src/test/datascience/ipywidgets/cdnWidgetScriptSourceProvider.unit.test.ts +++ b/src/test/datascience/ipywidgets/cdnWidgetScriptSourceProvider.unit.test.ts @@ -37,8 +37,11 @@ suite('Data Science - ipywidget - CDN', () => { suite(localLaunch ? 'Local Jupyter Server' : 'Remote Jupyter Server', () => { setup(() => { const connection: IConnection = { + type: 'jupyter', baseUrl: '', localProcExitCode: undefined, + valid: true, + displayName: '', disconnected: new EventEmitter().event, dispose: noop, hostName: '', diff --git a/src/test/datascience/ipywidgets/ipyWidgetScriptSourceProvider.unit.test.ts b/src/test/datascience/ipywidgets/ipyWidgetScriptSourceProvider.unit.test.ts index 5f404389ab7a..744c21d405a6 100644 --- a/src/test/datascience/ipywidgets/ipyWidgetScriptSourceProvider.unit.test.ts +++ b/src/test/datascience/ipywidgets/ipyWidgetScriptSourceProvider.unit.test.ts @@ -73,6 +73,9 @@ suite('xxxData Science - ipywidget - Widget Script Source Provider', () => { suite(localLaunch ? 'Local Jupyter Server' : 'Remote Jupyter Server', () => { setup(() => { const connection: IConnection = { + type: 'jupyter', + valid: true, + displayName: '', baseUrl: '', localProcExitCode: undefined, disconnected: new EventEmitter().event, diff --git a/src/test/datascience/jupyter/kernels/kernelSwitcher.unit.test.ts b/src/test/datascience/jupyter/kernels/kernelSwitcher.unit.test.ts index 951c8642dc42..6ad434cc5451 100644 --- a/src/test/datascience/jupyter/kernels/kernelSwitcher.unit.test.ts +++ b/src/test/datascience/jupyter/kernels/kernelSwitcher.unit.test.ts @@ -16,16 +16,15 @@ import { Architecture } from '../../../../client/common/utils/platform'; import { JupyterSessionStartError } from '../../../../client/datascience/baseJupyterSession'; import { Commands } from '../../../../client/datascience/constants'; import { JupyterNotebookBase } from '../../../../client/datascience/jupyter/jupyterNotebook'; -import { JupyterServerWrapper } from '../../../../client/datascience/jupyter/jupyterServerWrapper'; import { JupyterSessionManagerFactory } from '../../../../client/datascience/jupyter/jupyterSessionManagerFactory'; import { KernelSelector } from '../../../../client/datascience/jupyter/kernels/kernelSelector'; import { KernelSwitcher } from '../../../../client/datascience/jupyter/kernels/kernelSwitcher'; import { LiveKernelModel } from '../../../../client/datascience/jupyter/kernels/types'; import { + IConnection, IJupyterKernelSpec, IJupyterSessionManagerFactory, - INotebook, - INotebookServer + INotebook } from '../../../../client/datascience/types'; import { InterpreterType, PythonInterpreter } from '../../../../client/interpreter/contracts'; import { noop } from '../../../core'; @@ -38,14 +37,14 @@ suite('Data Science - Kernel Switcher', () => { let kernelSelector: KernelSelector; let appShell: IApplicationShell; let notebook: INotebook; - let notebookServer: INotebookServer; + let connection: IConnection; let currentKernel: IJupyterKernelSpec | LiveKernelModel; let selectedKernel: LiveKernelModel; let selectedKernelSecondTime: LiveKernelModel; let selectedInterpreter: PythonInterpreter; let settings: IPythonSettings; setup(() => { - notebookServer = mock(JupyterServerWrapper); + connection = mock(); settings = mock(PythonSettings); currentKernel = { lastActivityTime: new Date(), @@ -83,7 +82,7 @@ suite('Data Science - Kernel Switcher', () => { // tslint:disable-next-line: no-any when(settings.datascience).thenReturn({} as any); - when(notebook.server).thenReturn(instance(notebookServer)); + when(notebook.connection).thenReturn(instance(connection)); when(configService.getSettings(anything())).thenReturn(instance(settings)); kernelSwitcher = new KernelSwitcher( instance(configService), @@ -100,19 +99,23 @@ suite('Data Science - Kernel Switcher', () => { // tslint:disable-next-line: max-func-body-length suite(isLocalConnection ? 'Local Connection' : 'Remote Connection', () => { setup(() => { - when(notebookServer.getConnectionInfo()).thenReturn({ + const jupyterConnection: IConnection = { + type: 'jupyter', localLaunch: isLocalConnection, baseUrl: '', disconnected: new EventEmitter().event, hostName: '', token: '', localProcExitCode: 0, + valid: true, + displayName: '', dispose: noop - }); + }; + when(notebook.connection).thenReturn(jupyterConnection); }); teardown(() => { // We should have checked if it was a local connection. - verify(notebookServer.getConnectionInfo()).atLeast(1); + verify(notebook.connection).atLeast(1); }); [ diff --git a/src/test/datascience/mockJupyterNotebook.ts b/src/test/datascience/mockJupyterNotebook.ts index 75057536b218..aa23d23a53a7 100644 --- a/src/test/datascience/mockJupyterNotebook.ts +++ b/src/test/datascience/mockJupyterNotebook.ts @@ -11,13 +11,12 @@ import { LiveKernelModel } from '../../client/datascience/jupyter/kernels/types' import { ICell, ICellHashProvider, - IConnection, IGatherProvider, IJupyterKernelSpec, INotebook, INotebookCompletion, INotebookExecutionLogger, - INotebookServer, + INotebookProviderConnection, InterruptResult, KernelSocketInformation } from '../../client/datascience/types'; @@ -26,11 +25,8 @@ import { ServerStatus } from '../../datascience-ui/interactive-common/mainState' import { noop } from '../core'; export class MockJupyterNotebook implements INotebook { - public get server(): INotebookServer { - return this.owner; - } - public get connection(): IConnection { - throw new Error('Not implemented'); + public get connection(): INotebookProviderConnection | undefined { + return this.providerConnection; } public get identity(): Uri { return Uri.parse(Identifiers.InteractiveWindowIdentity); @@ -57,7 +53,7 @@ export class MockJupyterNotebook implements INotebook { public onKernelRestarted = new EventEmitter().event; private onStatusChangedEvent: EventEmitter | undefined; - constructor(private owner: INotebookServer) { + constructor(private providerConnection: INotebookProviderConnection | undefined) { noop(); } public registerIOPubListener( diff --git a/src/test/datascience/mockJupyterServer.ts b/src/test/datascience/mockJupyterServer.ts index 8ac8bff2683d..f92c1fe35e78 100644 --- a/src/test/datascience/mockJupyterServer.ts +++ b/src/test/datascience/mockJupyterServer.ts @@ -32,11 +32,11 @@ export class MockJupyterServer implements INotebookServer { } public async createNotebook(_resource: Uri): Promise { - return new MockJupyterNotebook(this); + return new MockJupyterNotebook(this.getConnectionInfo()); } public async getNotebook(_resource: Uri): Promise { - return new MockJupyterNotebook(this); + return new MockJupyterNotebook(this.getConnectionInfo()); } public async setMatplotLibStyle(_useDark: boolean): Promise { diff --git a/src/test/datascience/notebook.functional.test.ts b/src/test/datascience/notebook.functional.test.ts index b49a605dc5df..23858a12862d 100644 --- a/src/test/datascience/notebook.functional.test.ts +++ b/src/test/datascience/notebook.functional.test.ts @@ -1242,7 +1242,8 @@ plt.show()`, assert.ok(notebook, 'Server should have started on port 9975'); const hs = notebook as HostJupyterNotebook; // Check port number. Should have at least started with the one specified. - assert.ok(hs.server.getConnectionInfo()?.baseUrl.startsWith('http://localhost:99'), 'Port was not used'); + const jupyterConnectionInfo = hs.connection as IConnection; + assert.ok(jupyterConnectionInfo.baseUrl.startsWith('http://localhost:99'), 'Port was not used'); await verifySimple(hs, `a=1${os.EOL}a`, 1); } From d4f46241ab99aa9d6239245c3101f0c4fa7db513 Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Thu, 9 Apr 2020 09:53:47 -0700 Subject: [PATCH 014/725] Raw Notebook Provider (#11018) --- package.nls.json | 2 + src/client/common/utils/localize.ts | 8 + src/client/datascience/constants.ts | 2 + .../interactive-common/notebookProvider.ts | 158 ++++-- .../notebookServerProvider.ts | 6 +- .../jupyter/jupyterNotebookProvider.ts | 56 +++ .../liveshare/guestRawNotebookProvider.ts | 73 +++ .../liveshare/hostRawNotebookProvider.ts | 159 ++++++ .../datascience/raw-kernel/rawKernel.ts | 1 - .../raw-kernel/rawNotebookProvider.ts | 105 ++++ .../raw-kernel/rawNotebookProviderWrapper.ts | 98 ++++ src/client/datascience/serviceRegistry.ts | 462 +++++++++--------- src/client/datascience/types.ts | 26 +- .../datascience/dataScienceIocContainer.ts | 10 +- .../notebookProvider.unit.test.ts | 102 ++-- 15 files changed, 931 insertions(+), 337 deletions(-) create mode 100644 src/client/datascience/jupyter/jupyterNotebookProvider.ts create mode 100644 src/client/datascience/raw-kernel/liveshare/guestRawNotebookProvider.ts create mode 100644 src/client/datascience/raw-kernel/liveshare/hostRawNotebookProvider.ts create mode 100644 src/client/datascience/raw-kernel/rawNotebookProvider.ts create mode 100644 src/client/datascience/raw-kernel/rawNotebookProviderWrapper.ts diff --git a/package.nls.json b/package.nls.json index 8262076060c8..9da4caf938dc 100644 --- a/package.nls.json +++ b/package.nls.json @@ -126,6 +126,8 @@ "DataScience.jupyterSelfCertEnable": "Yes, connect anyways", "DataScience.jupyterSelfCertClose": "No, close the connection", "DataScience.jupyterServerCrashed": "Jupyter server crashed. Unable to connect. \r\nError code from jupyter: {0}", + "DataScience.rawConnectionDisplayName": "Direct kernel connection", + "DataScience.rawConnectionBrokenError": "Direct kernel connection broken", "DataScience.pythonInteractiveHelpLink": "Get more help", "DataScience.markdownHelpInstallingMissingDependencies": "See [https://aka.ms/pyaiinstall](https://aka.ms/pyaiinstall) for help on installing Jupyter and related dependencies.", "DataScience.importingFormat": "Importing {0}", diff --git a/src/client/common/utils/localize.ts b/src/client/common/utils/localize.ts index 8d5d91df58d6..bd7ad53ca41e 100644 --- a/src/client/common/utils/localize.ts +++ b/src/client/common/utils/localize.ts @@ -479,6 +479,14 @@ export namespace DataScience { 'DataScience.jupyterNotebookRemoteConnectSelfCertsFailed', 'Failed to connect to remote Jupyter notebook.\r\nSpecified server is using self signed certs. Enable Allow Unauthorized Remote Connection setting to connect anyways\r\n{0}\r\n{1}' ); + export const rawConnectionDisplayName = localize( + 'DataScience.rawConnectionDisplayName', + 'Direct kernel connection' + ); + export const rawConnectionBrokenError = localize( + 'DataScience.rawConnectionBrokenError', + 'Direct kernel connection broken' + ); export const jupyterServerCrashed = localize( 'DataScience.jupyterServerCrashed', 'Jupyter server crashed. Unable to connect. \r\nError code from jupyter: {0}' diff --git a/src/client/datascience/constants.ts b/src/client/datascience/constants.ts index 8090d3137e1c..fb15b06e75c2 100644 --- a/src/client/datascience/constants.ts +++ b/src/client/datascience/constants.ts @@ -397,6 +397,7 @@ export namespace Identifiers { export const EmptyFileName = '2DB9B899-6519-4E1B-88B0-FA728A274115'; export const GeneratedThemeName = 'ipython-theme'; // This needs to be all lower class and a valid class name. export const HistoryPurpose = 'history'; + export const RawPurpose = 'raw'; export const PingPurpose = 'ping'; export const MatplotLibDefaultParams = '_VSCode_defaultMatplotlib_Params'; export const EditCellId = '3D3AB152-ADC1-4501-B813-4B83B49B0C10'; @@ -443,6 +444,7 @@ export namespace LiveShare { export const InteractiveWindowProviderService = 'interactiveWindowProviderService'; export const GuestCheckerService = 'guestCheckerService'; export const LiveShareBroadcastRequest = 'broadcastRequest'; + export const RawNotebookProviderService = 'rawNotebookProviderSharedService'; export const ResponseLifetime = 15000; export const ResponseRange = 1000; // Range of time alloted to check if a response matches or not export const InterruptDefaultTimeout = 10000; diff --git a/src/client/datascience/interactive-common/notebookProvider.ts b/src/client/datascience/interactive-common/notebookProvider.ts index 230bed2d71e9..5bc4cc7b21ea 100644 --- a/src/client/datascience/interactive-common/notebookProvider.ts +++ b/src/client/datascience/interactive-common/notebookProvider.ts @@ -5,25 +5,31 @@ import { inject, injectable } from 'inversify'; import { EventEmitter, Uri } from 'vscode'; +import { LocalZMQKernel } from '../../common/experimentGroups'; +import { traceError, traceInfo } from '../../common/logger'; import { IFileSystem } from '../../common/platform/types'; -import { IDisposableRegistry } from '../../common/types'; +import { IConfigurationService, IDisposableRegistry, IExperimentsManager } from '../../common/types'; import { noop } from '../../common/utils/misc'; +import { sendTelemetryEvent } from '../../telemetry'; +import { Settings, Telemetry } from '../constants'; import { ConnectNotebookProviderOptions, GetNotebookOptions, IInteractiveWindowProvider, + IJupyterNotebookProvider, INotebook, INotebookEditor, INotebookEditorProvider, INotebookProvider, INotebookProviderConnection, - INotebookServerProvider + IRawNotebookProvider } from '../types'; @injectable() export class NotebookProvider implements INotebookProvider { private readonly notebooks = new Map>(); private _notebookCreated = new EventEmitter<{ identity: Uri; notebook: INotebook }>(); + private _zmqSupported: boolean | undefined; public get activeNotebooks() { return [...this.notebooks.values()]; } @@ -32,7 +38,10 @@ export class NotebookProvider implements INotebookProvider { @inject(INotebookEditorProvider) private readonly editorProvider: INotebookEditorProvider, @inject(IInteractiveWindowProvider) private readonly interactiveWindowProvider: IInteractiveWindowProvider, @inject(IDisposableRegistry) disposables: IDisposableRegistry, - @inject(INotebookServerProvider) private readonly serverProvider: INotebookServerProvider + @inject(IRawNotebookProvider) private readonly rawNotebookProvider: IRawNotebookProvider, + @inject(IJupyterNotebookProvider) private readonly jupyterNotebookProvider: IJupyterNotebookProvider, + @inject(IConfigurationService) private readonly configuration: IConfigurationService, + @inject(IExperimentsManager) private readonly experimentsManager: IExperimentsManager ) { disposables.push(editorProvider.onDidCloseNotebookEditor(this.onDidCloseNotebookEditor, this)); disposables.push( @@ -45,59 +54,118 @@ export class NotebookProvider implements INotebookProvider { // Disconnect from the specified provider public async disconnect(options: ConnectNotebookProviderOptions): Promise { - const server = await this.serverProvider.getOrCreateServer(options); - - return server?.dispose(); + // Only need to disconnect from actual jupyter servers + if (!(await this.rawKernelSupported())) { + return this.jupyterNotebookProvider.disconnect(options); + } } // Attempt to connect to our server provider, and if we do, return the connection info public async connect(options: ConnectNotebookProviderOptions): Promise { - const server = await this.serverProvider.getOrCreateServer(options); - - return server?.getConnectionInfo(); + // Connect to either a jupyter server or a stubbed out raw notebook "connection" + if (await this.rawKernelSupported()) { + return this.rawNotebookProvider.connect(); + } else { + return this.jupyterNotebookProvider.connect(options); + } } public async getOrCreateNotebook(options: GetNotebookOptions): Promise { - // Make sure we have a server - const server = await this.serverProvider.getOrCreateServer({ - getOnly: options.getOnly, - disableUI: options.disableUI - }); - if (server) { - // We could have multiple native editors opened for the same file/model. - const notebook = await server.getNotebook(options.identity); - if (notebook) { - return notebook; - } + const rawKernel = await this.rawKernelSupported(); + + // Check to see if our provider already has this notebook + const notebook = rawKernel + ? await this.rawNotebookProvider.getNotebook(options.identity) + : await this.jupyterNotebookProvider.getNotebook(options); + if (notebook) { + return notebook; + } - if (this.notebooks.get(options.identity.fsPath)) { - return this.notebooks.get(options.identity.fsPath)!!; + // Next check our own promise cache + if (this.notebooks.get(options.identity.fsPath)) { + return this.notebooks.get(options.identity.fsPath)!!; + } + + // We want to cache a Promise from the create functions + // but jupyterNotebookProvider.createNotebook can be undefined if the server is not available + // so check for our connection here first + if (!rawKernel) { + if (!(await this.jupyterNotebookProvider.connect(options))) { + return undefined; } + } + + // Finally create if needed + const promise = rawKernel + ? this.rawNotebookProvider.createNotebook(options.identity, options.identity, options.metadata) + : this.jupyterNotebookProvider.createNotebook(options); + + this.cacheNotebookPromise(options.identity, promise); + + return promise; + } + + // Check to see if we have all that we need for supporting raw kernel launch + private async rawKernelSupported(): Promise { + const zmqOk = await this.zmqSupported(); - const promise = server.createNotebook(options.identity, options.identity, options.metadata); - this.notebooks.set(options.identity.fsPath, promise); - - // Remove promise from cache if the same promise still exists. - const removeFromCache = () => { - const cachedPromise = this.notebooks.get(options.identity.fsPath); - if (cachedPromise === promise) { - this.notebooks.delete(options.identity.fsPath); - } - }; - - promise - .then((nb) => { - // If the notebook is disposed, remove from cache. - nb.onDisposed(removeFromCache); - this._notebookCreated.fire({ identity: options.identity, notebook: nb }); - }) - .catch(noop); - - // If promise fails, then remove the promise from cache. - promise.catch(removeFromCache); - - return promise; + return zmqOk && this.localLaunch() && this.experimentsManager.inExperiment(LocalZMQKernel.experiment) + ? true + : false; + } + + private localLaunch(): boolean { + const settings = this.configuration.getSettings(undefined); + const serverURI: string | undefined = settings.datascience.jupyterServerURI; + + if (!serverURI || serverURI.toLowerCase() === Settings.JupyterServerLocalLaunch) { + return true; + } + + return false; + } + + // Check to see if this machine supports our local ZMQ launching + private async zmqSupported(): Promise { + if (this._zmqSupported) { + return this._zmqSupported; } + + try { + await import('zeromq'); + traceInfo(`ZMQ install verified.`); + this._zmqSupported = true; + } catch (e) { + traceError(`Exception while attempting zmq :`, e); + sendTelemetryEvent(Telemetry.ZMQNotSupported); + this._zmqSupported = false; + } + + return this._zmqSupported; + } + + // Cache the promise that will return a notebook + private cacheNotebookPromise(identity: Uri, promise: Promise) { + this.notebooks.set(identity.fsPath, promise); + + // Remove promise from cache if the same promise still exists. + const removeFromCache = () => { + const cachedPromise = this.notebooks.get(identity.fsPath); + if (cachedPromise === promise) { + this.notebooks.delete(identity.fsPath); + } + }; + + promise + .then((nb) => { + // If the notebook is disposed, remove from cache. + nb.onDisposed(removeFromCache); + this._notebookCreated.fire({ identity: identity, notebook: nb }); + }) + .catch(noop); + + // If promise fails, then remove the promise from cache. + promise.catch(removeFromCache); } private async onDidCloseNotebookEditor(editor: INotebookEditor) { diff --git a/src/client/datascience/interactive-common/notebookServerProvider.ts b/src/client/datascience/interactive-common/notebookServerProvider.ts index 52d5dec5bcd8..5fe86cd4d2df 100644 --- a/src/client/datascience/interactive-common/notebookServerProvider.ts +++ b/src/client/datascience/interactive-common/notebookServerProvider.ts @@ -20,14 +20,14 @@ import { ProgressReporter } from '../progress/progressReporter'; import { GetServerOptions, IJupyterExecution, + IJupyterServerProvider, INotebook, INotebookServer, - INotebookServerOptions, - INotebookServerProvider + INotebookServerOptions } from '../types'; @injectable() -export class NotebookServerProvider implements INotebookServerProvider { +export class NotebookServerProvider implements IJupyterServerProvider { private serverPromise: Promise | undefined; private allowingUI = false; private _notebookCreated = new EventEmitter<{ identity: Uri; notebook: INotebook }>(); diff --git a/src/client/datascience/jupyter/jupyterNotebookProvider.ts b/src/client/datascience/jupyter/jupyterNotebookProvider.ts new file mode 100644 index 000000000000..ce099060ffdd --- /dev/null +++ b/src/client/datascience/jupyter/jupyterNotebookProvider.ts @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +import { inject, injectable } from 'inversify'; +import * as localize from '../../common/utils/localize'; +import { + ConnectNotebookProviderOptions, + GetNotebookOptions, + IConnection, + IJupyterNotebookProvider, + IJupyterServerProvider, + INotebook +} from '../types'; + +// When the NotebookProvider looks to create a notebook it uses this class to create a Jupyter notebook +@injectable() +export class JupyterNotebookProvider implements IJupyterNotebookProvider { + constructor(@inject(IJupyterServerProvider) private readonly serverProvider: IJupyterServerProvider) {} + + public async disconnect(options: ConnectNotebookProviderOptions): Promise { + const server = await this.serverProvider.getOrCreateServer(options); + + return server?.dispose(); + } + + public async connect(options: ConnectNotebookProviderOptions): Promise { + const server = await this.serverProvider.getOrCreateServer(options); + return server?.getConnectionInfo(); + } + + public async createNotebook(options: GetNotebookOptions): Promise { + // Make sure we have a server + const server = await this.serverProvider.getOrCreateServer({ + getOnly: options.getOnly, + disableUI: options.disableUI + }); + + if (server) { + return server.createNotebook(options.identity, options.identity, options.metadata); + } + // We want createNotebook to always return a notebook promise, so if we don't have a server + // here throw our generic server disposed message that we use in server creatio n + throw new Error(localize.DataScience.sessionDisposed()); + } + public async getNotebook(options: GetNotebookOptions): Promise { + const server = await this.serverProvider.getOrCreateServer({ + getOnly: options.getOnly, + disableUI: options.disableUI + }); + if (server) { + return server.getNotebook(options.identity); + } + } +} diff --git a/src/client/datascience/raw-kernel/liveshare/guestRawNotebookProvider.ts b/src/client/datascience/raw-kernel/liveshare/guestRawNotebookProvider.ts new file mode 100644 index 000000000000..6ce9d9bc5c2c --- /dev/null +++ b/src/client/datascience/raw-kernel/liveshare/guestRawNotebookProvider.ts @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +'use strict'; +import { nbformat } from '@jupyterlab/coreutils'; +import { Uri } from 'vscode'; +import { CancellationToken } from 'vscode-jsonrpc'; +import * as vsls from 'vsls/vscode'; +import { IApplicationShell, ILiveShareApi, IWorkspaceService } from '../../../common/application/types'; +import { IFileSystem } from '../../../common/platform/types'; +import { IAsyncDisposableRegistry, IConfigurationService, IDisposableRegistry, Resource } from '../../../common/types'; +import { IServiceContainer } from '../../../ioc/types'; +import { LiveShare } from '../../constants'; +import { + LiveShareParticipantDefault, + LiveShareParticipantGuest +} from '../../jupyter/liveshare/liveShareParticipantMixin'; +import { ILiveShareParticipant } from '../../jupyter/liveshare/types'; +import { INotebook, IRawConnection, IRawNotebookProvider } from '../../types'; + +export class GuestRawNotebookProvider + extends LiveShareParticipantGuest(LiveShareParticipantDefault, LiveShare.RawNotebookProviderService) + implements IRawNotebookProvider, ILiveShareParticipant { + constructor( + liveShare: ILiveShareApi, + _disposableRegistry: IDisposableRegistry, + _asyncRegistry: IAsyncDisposableRegistry, + _configService: IConfigurationService, + _workspaceService: IWorkspaceService, + _appShell: IApplicationShell, + _fs: IFileSystem, + _serviceContainer: IServiceContainer + ) { + super(liveShare); + } + + public async supported(): Promise { + // For now just false, but when implemented will message the host + return false; + } + + public async createNotebook( + _identity: Uri, + _resource: Resource, + _notebookMetadata: nbformat.INotebookMetadata, + _cancelToken: CancellationToken + ): Promise { + throw new Error('Not implemented'); + } + + public connect(): Promise { + throw new Error('Not implemented'); + } + + public async onSessionChange(_api: vsls.LiveShare | null): Promise { + // Not implemented yet + } + + public async getNotebook(_resource: Uri): Promise { + throw new Error('Not implemented'); + } + + public async shutdown(): Promise { + throw new Error('Not implemented'); + } + + public dispose(): Promise { + throw new Error('Not implemented'); + } + + public async onAttach(_api: vsls.LiveShare | null): Promise { + // Not implemented yet + } +} diff --git a/src/client/datascience/raw-kernel/liveshare/hostRawNotebookProvider.ts b/src/client/datascience/raw-kernel/liveshare/hostRawNotebookProvider.ts new file mode 100644 index 000000000000..dcdecff882be --- /dev/null +++ b/src/client/datascience/raw-kernel/liveshare/hostRawNotebookProvider.ts @@ -0,0 +1,159 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +'use strict'; +import '../../../common/extensions'; + +import * as vscode from 'vscode'; +import { CancellationToken } from 'vscode-jsonrpc'; +import * as vsls from 'vsls/vscode'; + +import { nbformat } from '@jupyterlab/coreutils'; +import { IApplicationShell, ILiveShareApi, IWorkspaceService } from '../../../common/application/types'; +import { traceInfo } from '../../../common/logger'; +import { IFileSystem } from '../../../common/platform/types'; +import { IAsyncDisposableRegistry, IConfigurationService, IDisposableRegistry, Resource } from '../../../common/types'; +import { createDeferred } from '../../../common/utils/async'; +import { IServiceContainer } from '../../../ioc/types'; +import { Identifiers, LiveShare, Settings } from '../../constants'; +import { HostJupyterNotebook } from '../../jupyter/liveshare/hostJupyterNotebook'; +import { LiveShareParticipantHost } from '../../jupyter/liveshare/liveShareParticipantMixin'; +import { IRoleBasedObject } from '../../jupyter/liveshare/roleBasedFactory'; +import { INotebook, INotebookExecutionInfo, INotebookExecutionLogger, IRawNotebookProvider } from '../../types'; +import { EnchannelJMPConnection } from '../enchannelJMPConnection'; +import { RawJupyterSession } from '../rawJupyterSession'; +import { RawNotebookProviderBase } from '../rawNotebookProvider'; + +// tslint:disable-next-line: no-require-imports +// tslint:disable:no-any + +export class HostRawNotebookProvider + extends LiveShareParticipantHost(RawNotebookProviderBase, LiveShare.RawNotebookProviderService) + implements IRoleBasedObject, IRawNotebookProvider { + private disposed = false; + constructor( + private liveShare: ILiveShareApi, + private disposableRegistry: IDisposableRegistry, + asyncRegistry: IAsyncDisposableRegistry, + private configService: IConfigurationService, + private workspaceService: IWorkspaceService, + private appShell: IApplicationShell, + private fs: IFileSystem, + private serviceContainer: IServiceContainer + ) { + super(liveShare, asyncRegistry); + } + + public async dispose(): Promise { + if (!this.disposed) { + this.disposed = true; + await super.dispose(); + } + } + + public async onAttach(_api: vsls.LiveShare | null): Promise { + // Not implemented yet + } + + public async onSessionChange(_api: vsls.LiveShare | null): Promise { + // Not implemented yet + } + + public async onDetach(_api: vsls.LiveShare | null): Promise { + // Not implemented yet + } + + public async waitForServiceName(): Promise { + return 'Not implemented'; + } + + protected async createNotebookInstance( + resource: Resource, + identity: vscode.Uri, + notebookMetadata?: nbformat.INotebookMetadata, + cancelToken?: CancellationToken + ): Promise { + throw new Error('Not implemented'); + // RAWKERNEL: Hack to create session, uncomment throw and update ci to connect to a running kernel + const ci = { + version: 0, + transport: 'tcp', + ip: '127.0.0.1', + shell_port: 51065, + iopub_port: 51066, + stdin_port: 51067, + hb_port: 51069, + control_port: 51068, + signature_scheme: 'hmac-sha256', + key: '9a4f68cd-b5e4887e4b237ea4c91c265c' + }; + const rawSession = new RawJupyterSession(new EnchannelJMPConnection()); + try { + await rawSession.connect(ci); + } finally { + if (!rawSession.isConnected) { + await rawSession.dispose(); + } + } + + const notebookPromise = createDeferred(); + this.setNotebook(identity, notebookPromise.promise); + + try { + // Get the execution info for our notebook + const info = this.getExecutionInfo(resource, notebookMetadata); + + if (rawSession.isConnected) { + // Create our notebook + const notebook = new HostJupyterNotebook( + this.liveShare, + rawSession, + this.configService, + this.disposableRegistry, + info, + this.serviceContainer.getAll(INotebookExecutionLogger), + resource, + identity, + this.getDisposedError.bind(this), + this.workspaceService, + this.appShell, + this.fs + ); + + // Wait for it to be ready + traceInfo(`Waiting for idle (session) ${this.id}`); + const idleTimeout = this.configService.getSettings().datascience.jupyterLaunchTimeout; + await notebook.waitForIdle(idleTimeout); + + // Run initial setup + await notebook.initialize(cancelToken); + + traceInfo(`Finished connecting ${this.id}`); + + notebookPromise.resolve(notebook); + } else { + notebookPromise.reject(this.getDisposedError()); + } + } catch (ex) { + // If there's an error, then reject the promise that is returned. + // This original promise must be rejected as it is cached (check `setNotebook`). + notebookPromise.reject(ex); + } + + return notebookPromise.promise; + } + + // RAWKERNEL: Not the real execution info, just stub it out for now + private getExecutionInfo( + _resource: Resource, + _notebookMetadata?: nbformat.INotebookMetadata + ): INotebookExecutionInfo { + return { + connectionInfo: this.getConnection(), + uri: Settings.JupyterServerLocalLaunch, + interpreter: undefined, + kernelSpec: undefined, + workingDir: undefined, + purpose: Identifiers.RawPurpose + }; + } +} diff --git a/src/client/datascience/raw-kernel/rawKernel.ts b/src/client/datascience/raw-kernel/rawKernel.ts index f55626419a22..b842049474ab 100644 --- a/src/client/datascience/raw-kernel/rawKernel.ts +++ b/src/client/datascience/raw-kernel/rawKernel.ts @@ -460,7 +460,6 @@ export class RawKernel implements Kernel.IKernel { // Handle a new message arriving from JMP connection private async handleMessage(message: KernelMessage.IMessage): Promise { - // IANHU: CONVERT TO USING ONE REQUIRE? // tslint:disable-next-line:no-require-imports const jupyterLab = require('@jupyterlab/services') as typeof import('@jupyterlab/services'); diff --git a/src/client/datascience/raw-kernel/rawNotebookProvider.ts b/src/client/datascience/raw-kernel/rawNotebookProvider.ts new file mode 100644 index 000000000000..8cfee8c5b0c0 --- /dev/null +++ b/src/client/datascience/raw-kernel/rawNotebookProvider.ts @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +'use strict'; +import { nbformat } from '@jupyterlab/coreutils'; +import * as uuid from 'uuid/v4'; +import { Event, EventEmitter, Uri } from 'vscode'; +import { CancellationToken } from 'vscode-jsonrpc'; +import { ILiveShareApi } from '../../common/application/types'; +import '../../common/extensions'; +import { traceInfo } from '../../common/logger'; +import { IAsyncDisposableRegistry, Resource } from '../../common/types'; +import * as localize from '../../common/utils/localize'; +import { noop } from '../../common/utils/misc'; +import { INotebook, IRawConnection, IRawNotebookProvider } from '../types'; + +class RawConnection implements IRawConnection { + public readonly type = 'raw'; + public readonly localLaunch = true; + public readonly valid = true; + public readonly displayName = localize.DataScience.rawConnectionDisplayName(); + private eventEmitter: EventEmitter = new EventEmitter(); + + public dispose() { + noop(); + } + public get disconnected(): Event { + return this.eventEmitter.event; + } +} + +export class RawNotebookProviderBase implements IRawNotebookProvider { + public get id(): string { + return this._id; + } + // Keep track of the notebooks that we have provided + private notebooks = new Map>(); + private rawConnection = new RawConnection(); + private _id = uuid(); + + constructor(_liveShare: ILiveShareApi, private asyncRegistry: IAsyncDisposableRegistry) { + this.asyncRegistry.push(this); + } + + public connect(): Promise { + return Promise.resolve(this.rawConnection); + } + + public async createNotebook( + identity: Uri, + resource: Resource, + notebookMetadata: nbformat.INotebookMetadata, + cancelToken: CancellationToken + ): Promise { + return this.createNotebookInstance(resource, identity, notebookMetadata, cancelToken); + } + + public async getNotebook(identity: Uri): Promise { + return this.notebooks.get(identity.toString()); + } + + public async dispose(): Promise { + traceInfo(`Shutting down notebooks for ${this.id}`); + const notebooks = await Promise.all([...this.notebooks.values()]); + await Promise.all(notebooks.map(n => n?.dispose())); + } + + // This may be a bit of a noop in the raw case + public getDisposedError(): Error { + return new Error(localize.DataScience.rawConnectionBrokenError()); + } + + protected getConnection(): IRawConnection { + return this.rawConnection; + } + + protected setNotebook(identity: Uri, notebook: Promise) { + const removeNotebook = () => { + if (this.notebooks.get(identity.toString()) === notebook) { + this.notebooks.delete(identity.toString()); + } + }; + + notebook + .then(nb => { + const oldDispose = nb.dispose; + nb.dispose = () => { + this.notebooks.delete(identity.toString()); + return oldDispose(); + }; + }) + .catch(removeNotebook); + + // Save the notebook + this.notebooks.set(identity.toString(), notebook); + } + + protected createNotebookInstance( + _resource: Resource, + _identity: Uri, + _notebookMetadata?: nbformat.INotebookMetadata, + _cancelToken?: CancellationToken + ): Promise { + throw new Error('You forgot to override createNotebookInstance'); + } +} diff --git a/src/client/datascience/raw-kernel/rawNotebookProviderWrapper.ts b/src/client/datascience/raw-kernel/rawNotebookProviderWrapper.ts new file mode 100644 index 000000000000..7ff7bbba5e6b --- /dev/null +++ b/src/client/datascience/raw-kernel/rawNotebookProviderWrapper.ts @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +'use strict'; +import { nbformat } from '@jupyterlab/coreutils'; +import { inject, injectable } from 'inversify'; +import { Uri } from 'vscode'; +import { CancellationToken } from 'vscode-jsonrpc'; +import * as vsls from 'vsls/vscode'; +import { IApplicationShell, ILiveShareApi, IWorkspaceService } from '../../common/application/types'; +import '../../common/extensions'; +import { IFileSystem } from '../../common/platform/types'; +import { IAsyncDisposableRegistry, IConfigurationService, IDisposableRegistry, Resource } from '../../common/types'; +import { IServiceContainer } from '../../ioc/types'; +import { IRoleBasedObject, RoleBasedFactory } from '../jupyter/liveshare/roleBasedFactory'; +import { ILiveShareHasRole } from '../jupyter/liveshare/types'; +import { INotebook, IRawConnection, IRawNotebookProvider } from '../types'; +import { GuestRawNotebookProvider } from './liveshare/guestRawNotebookProvider'; +import { HostRawNotebookProvider } from './liveshare/hostRawNotebookProvider'; + +interface IRawNotebookProviderInterface extends IRoleBasedObject, IRawNotebookProvider {} + +// tslint:disable:callable-types +type RawNotebookProviderClassType = { + new ( + liveShare: ILiveShareApi, + disposableRegistry: IDisposableRegistry, + asyncRegistry: IAsyncDisposableRegistry, + configService: IConfigurationService, + workspaceService: IWorkspaceService, + appShell: IApplicationShell, + fs: IFileSystem, + serviceContainer: IServiceContainer + ): IRawNotebookProviderInterface; +}; +// tslint:enable:callable-types + +// This class wraps either a HostRawNotebookProvider or a GuestRawNotebookProvider based on the liveshare state. It abstracts +// out the live share specific parts. +@injectable() +export class RawNotebookProviderWrapper implements IRawNotebookProvider, ILiveShareHasRole { + private serverFactory: RoleBasedFactory; + + constructor( + @inject(ILiveShareApi) liveShare: ILiveShareApi, + @inject(IDisposableRegistry) disposableRegistry: IDisposableRegistry, + @inject(IAsyncDisposableRegistry) asyncRegistry: IAsyncDisposableRegistry, + @inject(IConfigurationService) configService: IConfigurationService, + @inject(IWorkspaceService) workspaceService: IWorkspaceService, + @inject(IApplicationShell) appShell: IApplicationShell, + @inject(IFileSystem) fs: IFileSystem, + @inject(IServiceContainer) serviceContainer: IServiceContainer + ) { + // The server factory will create the appropriate HostRawNotebookProvider or GuestRawNotebookProvider based on + // the liveshare state. + this.serverFactory = new RoleBasedFactory( + liveShare, + HostRawNotebookProvider, + GuestRawNotebookProvider, + liveShare, + disposableRegistry, + asyncRegistry, + configService, + workspaceService, + appShell, + fs, + serviceContainer + ); + } + + public get role(): vsls.Role { + return this.serverFactory.role; + } + + public async connect(): Promise { + const notebookProvider = await this.serverFactory.get(); + return notebookProvider.connect(); + } + + public async createNotebook( + identity: Uri, + resource: Resource, + notebookMetadata: nbformat.INotebookMetadata, + cancelToken: CancellationToken + ): Promise { + const notebookProvider = await this.serverFactory.get(); + return notebookProvider.createNotebook(identity, resource, notebookMetadata, cancelToken); + } + + public async getNotebook(identity: Uri): Promise { + const notebookProvider = await this.serverFactory.get(); + return notebookProvider.getNotebook(identity); + } + + public async dispose(): Promise { + const server = await this.serverFactory.get(); + return server.dispose(); + } +} diff --git a/src/client/datascience/serviceRegistry.ts b/src/client/datascience/serviceRegistry.ts index 96b847d6b4d1..205316741a78 100644 --- a/src/client/datascience/serviceRegistry.ts +++ b/src/client/datascience/serviceRegistry.ts @@ -1,228 +1,234 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. -'use strict'; -import { IExtensionSingleActivationService } from '../activation/types'; -import { IApplicationEnvironment, IWorkspaceService } from '../common/application/types'; -import { UseCustomEditorApi } from '../common/constants'; -import { IServiceManager } from '../ioc/types'; -import { Activation } from './activation'; -import { CodeCssGenerator } from './codeCssGenerator'; -import { JupyterCommandLineSelectorCommand } from './commands/commandLineSelector'; -import { CommandRegistry } from './commands/commandRegistry'; -import { KernelSwitcherCommand } from './commands/kernelSwitcher'; -import { JupyterServerSelectorCommand } from './commands/serverSelector'; -import { ActiveEditorContextService } from './context/activeEditorContext'; -import { DataViewer } from './data-viewing/dataViewer'; -import { DataViewerDependencyService } from './data-viewing/dataViewerDependencyService'; -import { DataViewerProvider } from './data-viewing/dataViewerProvider'; -import { DataScience } from './datascience'; -import { DataScienceSurveyBannerLogger } from './dataScienceSurveyBanner'; -import { DebugLocationTrackerFactory } from './debugLocationTrackerFactory'; -import { CellHashProvider } from './editor-integration/cellhashprovider'; -import { CodeLensFactory } from './editor-integration/codeLensFactory'; -import { DataScienceCodeLensProvider } from './editor-integration/codelensprovider'; -import { CodeWatcher } from './editor-integration/codewatcher'; -import { Decorator } from './editor-integration/decorator'; -import { DataScienceErrorHandler } from './errorHandler/errorHandler'; -import { GatherListener } from './gather/gatherListener'; -import { GatherLogger } from './gather/gatherLogger'; -import { DebugListener } from './interactive-common/debugListener'; -import { IntellisenseProvider } from './interactive-common/intellisense/intellisenseProvider'; -import { LinkProvider } from './interactive-common/linkProvider'; -import { NotebookProvider } from './interactive-common/notebookProvider'; -import { NotebookServerProvider } from './interactive-common/notebookServerProvider'; -import { ShowPlotListener } from './interactive-common/showPlotListener'; -import { AutoSaveService } from './interactive-ipynb/autoSaveService'; -import { NativeEditor } from './interactive-ipynb/nativeEditor'; -import { NativeEditorCommandListener } from './interactive-ipynb/nativeEditorCommandListener'; -import { NativeEditorOldWebView } from './interactive-ipynb/nativeEditorOldWebView'; -import { NativeEditorProvider } from './interactive-ipynb/nativeEditorProvider'; -import { NativeEditorProviderOld } from './interactive-ipynb/nativeEditorProviderOld'; -import { NativeEditorStorage } from './interactive-ipynb/nativeEditorStorage'; -import { NativeEditorSynchronizer } from './interactive-ipynb/nativeEditorSynchronizer'; -import { InteractiveWindow } from './interactive-window/interactiveWindow'; -import { InteractiveWindowCommandListener } from './interactive-window/interactiveWindowCommandListener'; -import { InteractiveWindowProvider } from './interactive-window/interactiveWindowProvider'; -import { IPyWidgetHandler } from './ipywidgets/ipywidgetHandler'; -import { IPyWidgetMessageDispatcherFactory } from './ipywidgets/ipyWidgetMessageDispatcherFactory'; -import { IPyWidgetScriptSource } from './ipywidgets/ipyWidgetScriptSource'; -import { JupyterCommandLineSelector } from './jupyter/commandLineSelector'; -import { JupyterCommandFactory } from './jupyter/interpreter/jupyterCommand'; -import { JupyterCommandFinder } from './jupyter/interpreter/jupyterCommandFinder'; -import { JupyterCommandInterpreterDependencyService } from './jupyter/interpreter/jupyterCommandInterpreterDependencyService'; -import { JupyterCommandFinderInterpreterExecutionService } from './jupyter/interpreter/jupyterCommandInterpreterExecutionService'; -import { JupyterInterpreterDependencyService } from './jupyter/interpreter/jupyterInterpreterDependencyService'; -import { JupyterInterpreterOldCacheStateStore } from './jupyter/interpreter/jupyterInterpreterOldCacheStateStore'; -import { JupyterInterpreterSelectionCommand } from './jupyter/interpreter/jupyterInterpreterSelectionCommand'; -import { JupyterInterpreterSelector } from './jupyter/interpreter/jupyterInterpreterSelector'; -import { JupyterInterpreterService } from './jupyter/interpreter/jupyterInterpreterService'; -import { JupyterInterpreterStateStore } from './jupyter/interpreter/jupyterInterpreterStateStore'; -import { JupyterInterpreterSubCommandExecutionService } from './jupyter/interpreter/jupyterInterpreterSubCommandExecutionService'; -import { CellOutputMimeTypeTracker } from './jupyter/jupyterCellOutputMimeTypeTracker'; -import { JupyterDebugger } from './jupyter/jupyterDebugger'; -import { JupyterExecutionFactory } from './jupyter/jupyterExecutionFactory'; -import { JupyterExporter } from './jupyter/jupyterExporter'; -import { JupyterImporter } from './jupyter/jupyterImporter'; -import { JupyterPasswordConnect } from './jupyter/jupyterPasswordConnect'; -import { JupyterServerWrapper } from './jupyter/jupyterServerWrapper'; -import { JupyterSessionManagerFactory } from './jupyter/jupyterSessionManagerFactory'; -import { JupyterVariables } from './jupyter/jupyterVariables'; -import { KernelSelectionProvider } from './jupyter/kernels/kernelSelections'; -import { KernelSelector } from './jupyter/kernels/kernelSelector'; -import { KernelService } from './jupyter/kernels/kernelService'; -import { KernelSwitcher } from './jupyter/kernels/kernelSwitcher'; -import { NotebookStarter } from './jupyter/notebookStarter'; -import { ServerPreload } from './jupyter/serverPreload'; -import { JupyterServerSelector } from './jupyter/serverSelector'; -import { KernelFinder } from './kernel-launcher/kernelFinder'; -import { KernelLauncher } from './kernel-launcher/kernelLauncher'; -import { IKernelFinder, IKernelLauncher } from './kernel-launcher/types'; -import { PlotViewer } from './plotting/plotViewer'; -import { PlotViewerProvider } from './plotting/plotViewerProvider'; -import { PreWarmActivatedJupyterEnvironmentVariables } from './preWarmVariables'; -import { ProgressReporter } from './progress/progressReporter'; -import { EnchannelJMPConnection } from './raw-kernel/enchannelJMPConnection'; -import { StatusProvider } from './statusProvider'; -import { ThemeFinder } from './themeFinder'; -import { - ICellHashListener, - ICellHashProvider, - ICodeCssGenerator, - ICodeLensFactory, - ICodeWatcher, - IDataScience, - IDataScienceCodeLensProvider, - IDataScienceCommandListener, - IDataScienceErrorHandler, - IDataViewer, - IDataViewerProvider, - IDebugLocationTracker, - IGatherLogger, - IGatherProvider, - IInteractiveWindow, - IInteractiveWindowListener, - IInteractiveWindowProvider, - IJMPConnection, - IJupyterCommandFactory, - IJupyterDebugger, - IJupyterExecution, - IJupyterInterpreterDependencyManager, - IJupyterPasswordConnect, - IJupyterSessionManagerFactory, - IJupyterSubCommandExecutionService, - IJupyterVariables, - INotebookEditor, - INotebookEditorProvider, - INotebookExecutionLogger, - INotebookExporter, - INotebookImporter, - INotebookProvider, - INotebookServer, - INotebookServerProvider, - INotebookStorage, - IPlotViewer, - IPlotViewerProvider, - IStatusProvider, - IThemeFinder -} from './types'; - -// README: Did you make sure "dataScienceIocContainer.ts" has also been updated appropriately? - -// tslint:disable-next-line: max-func-body-length -export function registerTypes(serviceManager: IServiceManager) { - const useCustomEditorApi = serviceManager.get(IApplicationEnvironment).packageJson.enableProposedApi; - serviceManager.addSingletonInstance(UseCustomEditorApi, useCustomEditorApi); - - serviceManager.add(ICellHashProvider, CellHashProvider, undefined, [INotebookExecutionLogger]); - serviceManager.add(ICodeWatcher, CodeWatcher); - serviceManager.addSingleton(IDataScienceErrorHandler, DataScienceErrorHandler); - serviceManager.add(IDataViewer, DataViewer); - serviceManager.add(IInteractiveWindow, InteractiveWindow); - serviceManager.add(IInteractiveWindowListener, AutoSaveService); - serviceManager.add(IInteractiveWindowListener, DebugListener); - serviceManager.add(IInteractiveWindowListener, GatherListener); - serviceManager.add(IInteractiveWindowListener, IntellisenseProvider); - serviceManager.add(IInteractiveWindowListener, LinkProvider); - serviceManager.add(IInteractiveWindowListener, ShowPlotListener); - serviceManager.add(IInteractiveWindowListener, IPyWidgetHandler); - serviceManager.add(IInteractiveWindowListener, IPyWidgetScriptSource); - serviceManager.add(IJupyterCommandFactory, JupyterCommandFactory); - serviceManager.add(INotebookEditor, useCustomEditorApi ? NativeEditor : NativeEditorOldWebView); - serviceManager.add(INotebookExporter, JupyterExporter); - serviceManager.add(INotebookImporter, JupyterImporter); - serviceManager.add(INotebookServer, JupyterServerWrapper); - serviceManager.add(INotebookStorage, NativeEditorStorage); - serviceManager.add(IPlotViewer, PlotViewer); - serviceManager.addSingleton(IKernelLauncher, KernelLauncher); - serviceManager.addSingleton(IKernelFinder, KernelFinder); - serviceManager.addSingleton(ActiveEditorContextService, ActiveEditorContextService); - serviceManager.addSingleton(CellOutputMimeTypeTracker, CellOutputMimeTypeTracker, undefined, [IExtensionSingleActivationService, INotebookExecutionLogger]); - serviceManager.addSingleton(CommandRegistry, CommandRegistry); - serviceManager.addSingleton(DataViewerDependencyService, DataViewerDependencyService); - serviceManager.addSingleton(ICodeCssGenerator, CodeCssGenerator); - serviceManager.addSingleton(ICodeLensFactory, CodeLensFactory, undefined, [IInteractiveWindowListener]); - serviceManager.addSingleton(IDataScience, DataScience); - serviceManager.addSingleton(IDataScienceCodeLensProvider, DataScienceCodeLensProvider); - serviceManager.addSingleton(IDataScienceCommandListener, InteractiveWindowCommandListener); - serviceManager.addSingleton(IDataScienceCommandListener, NativeEditorCommandListener); - serviceManager.addSingleton(IDataViewerProvider, DataViewerProvider); - serviceManager.addSingleton(IDebugLocationTracker, DebugLocationTrackerFactory); - serviceManager.addSingleton(IExtensionSingleActivationService, Activation); - serviceManager.addSingleton(IExtensionSingleActivationService, Decorator); - serviceManager.addSingleton(IExtensionSingleActivationService, JupyterInterpreterSelectionCommand); - serviceManager.addSingleton(IExtensionSingleActivationService, PreWarmActivatedJupyterEnvironmentVariables); - serviceManager.addSingleton(IExtensionSingleActivationService, ServerPreload); - serviceManager.addSingleton(IInteractiveWindowListener, DataScienceSurveyBannerLogger); - serviceManager.addSingleton(IInteractiveWindowProvider, InteractiveWindowProvider); - serviceManager.addSingleton(IJupyterDebugger, JupyterDebugger, undefined, [ICellHashListener]); - serviceManager.addSingleton(IJupyterExecution, JupyterExecutionFactory); - serviceManager.addSingleton(IJupyterPasswordConnect, JupyterPasswordConnect); - serviceManager.addSingleton(IJupyterSessionManagerFactory, JupyterSessionManagerFactory); - serviceManager.addSingleton(IJupyterVariables, JupyterVariables); - serviceManager.addSingleton(INotebookEditorProvider, useCustomEditorApi ? NativeEditorProvider : NativeEditorProviderOld); - serviceManager.addSingleton(IPlotViewerProvider, PlotViewerProvider); - serviceManager.addSingleton(IStatusProvider, StatusProvider); - serviceManager.addSingleton(IThemeFinder, ThemeFinder); - serviceManager.addSingleton(JupyterCommandFinder, JupyterCommandFinder); - serviceManager.addSingleton(JupyterCommandLineSelector, JupyterCommandLineSelector); - serviceManager.addSingleton(JupyterCommandLineSelectorCommand, JupyterCommandLineSelectorCommand); - serviceManager.addSingleton(JupyterInterpreterDependencyService, JupyterInterpreterDependencyService); - serviceManager.addSingleton(JupyterInterpreterOldCacheStateStore, JupyterInterpreterOldCacheStateStore); - serviceManager.addSingleton(JupyterInterpreterSelector, JupyterInterpreterSelector); - serviceManager.addSingleton(JupyterInterpreterService, JupyterInterpreterService); - serviceManager.addSingleton(JupyterInterpreterStateStore, JupyterInterpreterStateStore); - serviceManager.addSingleton(JupyterServerSelector, JupyterServerSelector); - serviceManager.addSingleton(JupyterServerSelectorCommand, JupyterServerSelectorCommand); - serviceManager.addSingleton(KernelSelectionProvider, KernelSelectionProvider); - serviceManager.addSingleton(KernelSelector, KernelSelector); - serviceManager.addSingleton(KernelService, KernelService); - serviceManager.addSingleton(KernelSwitcher, KernelSwitcher); - serviceManager.addSingleton(KernelSwitcherCommand, KernelSwitcherCommand); - serviceManager.addSingleton(NotebookStarter, NotebookStarter); - serviceManager.addSingleton(ProgressReporter, ProgressReporter); - serviceManager.addSingleton(NativeEditorSynchronizer, NativeEditorSynchronizer); - serviceManager.addSingleton(INotebookProvider, NotebookProvider); - serviceManager.addSingleton(INotebookServerProvider, NotebookServerProvider); - serviceManager.add(IJMPConnection, EnchannelJMPConnection); - serviceManager.addSingleton(IPyWidgetMessageDispatcherFactory, IPyWidgetMessageDispatcherFactory); - - // Temporary code, to allow users to revert to the old behavior. - const cfg = serviceManager.get(IWorkspaceService).getConfiguration('python.dataScience', undefined); - if (cfg.get('useOldJupyterServer', false)) { - serviceManager.addSingleton(IJupyterInterpreterDependencyManager, JupyterCommandInterpreterDependencyService); - serviceManager.addSingleton(IJupyterSubCommandExecutionService, JupyterCommandFinderInterpreterExecutionService); - } else { - serviceManager.addSingleton(IJupyterInterpreterDependencyManager, JupyterInterpreterSubCommandExecutionService); - serviceManager.addSingleton(IJupyterSubCommandExecutionService, JupyterInterpreterSubCommandExecutionService); - } - - registerGatherTypes(serviceManager); -} - -export function registerGatherTypes(serviceManager: IServiceManager) { - // tslint:disable-next-line: no-require-imports - const gather = require('./gather/gather'); - - serviceManager.add(IGatherProvider, gather.GatherProvider); - serviceManager.add(IGatherLogger, GatherLogger, undefined, [INotebookExecutionLogger]); -} +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +'use strict'; +import { IExtensionSingleActivationService } from '../activation/types'; +import { IApplicationEnvironment, IWorkspaceService } from '../common/application/types'; +import { UseCustomEditorApi } from '../common/constants'; +import { IServiceManager } from '../ioc/types'; +import { Activation } from './activation'; +import { CodeCssGenerator } from './codeCssGenerator'; +import { JupyterCommandLineSelectorCommand } from './commands/commandLineSelector'; +import { CommandRegistry } from './commands/commandRegistry'; +import { KernelSwitcherCommand } from './commands/kernelSwitcher'; +import { JupyterServerSelectorCommand } from './commands/serverSelector'; +import { ActiveEditorContextService } from './context/activeEditorContext'; +import { DataViewer } from './data-viewing/dataViewer'; +import { DataViewerDependencyService } from './data-viewing/dataViewerDependencyService'; +import { DataViewerProvider } from './data-viewing/dataViewerProvider'; +import { DataScience } from './datascience'; +import { DataScienceSurveyBannerLogger } from './dataScienceSurveyBanner'; +import { DebugLocationTrackerFactory } from './debugLocationTrackerFactory'; +import { CellHashProvider } from './editor-integration/cellhashprovider'; +import { CodeLensFactory } from './editor-integration/codeLensFactory'; +import { DataScienceCodeLensProvider } from './editor-integration/codelensprovider'; +import { CodeWatcher } from './editor-integration/codewatcher'; +import { Decorator } from './editor-integration/decorator'; +import { DataScienceErrorHandler } from './errorHandler/errorHandler'; +import { GatherListener } from './gather/gatherListener'; +import { GatherLogger } from './gather/gatherLogger'; +import { DebugListener } from './interactive-common/debugListener'; +import { IntellisenseProvider } from './interactive-common/intellisense/intellisenseProvider'; +import { LinkProvider } from './interactive-common/linkProvider'; +import { NotebookProvider } from './interactive-common/notebookProvider'; +import { NotebookServerProvider } from './interactive-common/notebookServerProvider'; +import { ShowPlotListener } from './interactive-common/showPlotListener'; +import { AutoSaveService } from './interactive-ipynb/autoSaveService'; +import { NativeEditor } from './interactive-ipynb/nativeEditor'; +import { NativeEditorCommandListener } from './interactive-ipynb/nativeEditorCommandListener'; +import { NativeEditorOldWebView } from './interactive-ipynb/nativeEditorOldWebView'; +import { NativeEditorProvider } from './interactive-ipynb/nativeEditorProvider'; +import { NativeEditorProviderOld } from './interactive-ipynb/nativeEditorProviderOld'; +import { NativeEditorStorage } from './interactive-ipynb/nativeEditorStorage'; +import { NativeEditorSynchronizer } from './interactive-ipynb/nativeEditorSynchronizer'; +import { InteractiveWindow } from './interactive-window/interactiveWindow'; +import { InteractiveWindowCommandListener } from './interactive-window/interactiveWindowCommandListener'; +import { InteractiveWindowProvider } from './interactive-window/interactiveWindowProvider'; +import { IPyWidgetHandler } from './ipywidgets/ipywidgetHandler'; +import { IPyWidgetMessageDispatcherFactory } from './ipywidgets/ipyWidgetMessageDispatcherFactory'; +import { IPyWidgetScriptSource } from './ipywidgets/ipyWidgetScriptSource'; +import { JupyterCommandLineSelector } from './jupyter/commandLineSelector'; +import { JupyterCommandFactory } from './jupyter/interpreter/jupyterCommand'; +import { JupyterCommandFinder } from './jupyter/interpreter/jupyterCommandFinder'; +import { JupyterCommandInterpreterDependencyService } from './jupyter/interpreter/jupyterCommandInterpreterDependencyService'; +import { JupyterCommandFinderInterpreterExecutionService } from './jupyter/interpreter/jupyterCommandInterpreterExecutionService'; +import { JupyterInterpreterDependencyService } from './jupyter/interpreter/jupyterInterpreterDependencyService'; +import { JupyterInterpreterOldCacheStateStore } from './jupyter/interpreter/jupyterInterpreterOldCacheStateStore'; +import { JupyterInterpreterSelectionCommand } from './jupyter/interpreter/jupyterInterpreterSelectionCommand'; +import { JupyterInterpreterSelector } from './jupyter/interpreter/jupyterInterpreterSelector'; +import { JupyterInterpreterService } from './jupyter/interpreter/jupyterInterpreterService'; +import { JupyterInterpreterStateStore } from './jupyter/interpreter/jupyterInterpreterStateStore'; +import { JupyterInterpreterSubCommandExecutionService } from './jupyter/interpreter/jupyterInterpreterSubCommandExecutionService'; +import { CellOutputMimeTypeTracker } from './jupyter/jupyterCellOutputMimeTypeTracker'; +import { JupyterDebugger } from './jupyter/jupyterDebugger'; +import { JupyterExecutionFactory } from './jupyter/jupyterExecutionFactory'; +import { JupyterExporter } from './jupyter/jupyterExporter'; +import { JupyterImporter } from './jupyter/jupyterImporter'; +import { JupyterNotebookProvider } from './jupyter/jupyterNotebookProvider'; +import { JupyterPasswordConnect } from './jupyter/jupyterPasswordConnect'; +import { JupyterServerWrapper } from './jupyter/jupyterServerWrapper'; +import { JupyterSessionManagerFactory } from './jupyter/jupyterSessionManagerFactory'; +import { JupyterVariables } from './jupyter/jupyterVariables'; +import { KernelSelectionProvider } from './jupyter/kernels/kernelSelections'; +import { KernelSelector } from './jupyter/kernels/kernelSelector'; +import { KernelService } from './jupyter/kernels/kernelService'; +import { KernelSwitcher } from './jupyter/kernels/kernelSwitcher'; +import { NotebookStarter } from './jupyter/notebookStarter'; +import { ServerPreload } from './jupyter/serverPreload'; +import { JupyterServerSelector } from './jupyter/serverSelector'; +import { KernelFinder } from './kernel-launcher/kernelFinder'; +import { KernelLauncher } from './kernel-launcher/kernelLauncher'; +import { IKernelFinder, IKernelLauncher } from './kernel-launcher/types'; +import { PlotViewer } from './plotting/plotViewer'; +import { PlotViewerProvider } from './plotting/plotViewerProvider'; +import { PreWarmActivatedJupyterEnvironmentVariables } from './preWarmVariables'; +import { ProgressReporter } from './progress/progressReporter'; +import { EnchannelJMPConnection } from './raw-kernel/enchannelJMPConnection'; +import { RawNotebookProviderWrapper } from './raw-kernel/rawNotebookProviderWrapper'; +import { StatusProvider } from './statusProvider'; +import { ThemeFinder } from './themeFinder'; +import { + ICellHashListener, + ICellHashProvider, + ICodeCssGenerator, + ICodeLensFactory, + ICodeWatcher, + IDataScience, + IDataScienceCodeLensProvider, + IDataScienceCommandListener, + IDataScienceErrorHandler, + IDataViewer, + IDataViewerProvider, + IDebugLocationTracker, + IGatherLogger, + IGatherProvider, + IInteractiveWindow, + IInteractiveWindowListener, + IInteractiveWindowProvider, + IJMPConnection, + IJupyterCommandFactory, + IJupyterDebugger, + IJupyterExecution, + IJupyterInterpreterDependencyManager, + IJupyterNotebookProvider, + IJupyterPasswordConnect, + IJupyterServerProvider, + IJupyterSessionManagerFactory, + IJupyterSubCommandExecutionService, + IJupyterVariables, + INotebookEditor, + INotebookEditorProvider, + INotebookExecutionLogger, + INotebookExporter, + INotebookImporter, + INotebookProvider, + INotebookServer, + INotebookStorage, + IPlotViewer, + IPlotViewerProvider, + IRawNotebookProvider, + IStatusProvider, + IThemeFinder +} from './types'; + +// README: Did you make sure "dataScienceIocContainer.ts" has also been updated appropriately? + +// tslint:disable-next-line: max-func-body-length +export function registerTypes(serviceManager: IServiceManager) { + const useCustomEditorApi = serviceManager.get(IApplicationEnvironment).packageJson.enableProposedApi; + serviceManager.addSingletonInstance(UseCustomEditorApi, useCustomEditorApi); + + serviceManager.add(ICellHashProvider, CellHashProvider, undefined, [INotebookExecutionLogger]); + serviceManager.add(ICodeWatcher, CodeWatcher); + serviceManager.addSingleton(IDataScienceErrorHandler, DataScienceErrorHandler); + serviceManager.add(IDataViewer, DataViewer); + serviceManager.add(IInteractiveWindow, InteractiveWindow); + serviceManager.add(IInteractiveWindowListener, AutoSaveService); + serviceManager.add(IInteractiveWindowListener, DebugListener); + serviceManager.add(IInteractiveWindowListener, GatherListener); + serviceManager.add(IInteractiveWindowListener, IntellisenseProvider); + serviceManager.add(IInteractiveWindowListener, LinkProvider); + serviceManager.add(IInteractiveWindowListener, ShowPlotListener); + serviceManager.add(IInteractiveWindowListener, IPyWidgetHandler); + serviceManager.add(IInteractiveWindowListener, IPyWidgetScriptSource); + serviceManager.add(IJupyterCommandFactory, JupyterCommandFactory); + serviceManager.add(INotebookEditor, useCustomEditorApi ? NativeEditor : NativeEditorOldWebView); + serviceManager.add(INotebookExporter, JupyterExporter); + serviceManager.add(INotebookImporter, JupyterImporter); + serviceManager.add(INotebookServer, JupyterServerWrapper); + serviceManager.add(INotebookStorage, NativeEditorStorage); + serviceManager.addSingleton(IRawNotebookProvider, RawNotebookProviderWrapper); + serviceManager.addSingleton(IJupyterNotebookProvider, JupyterNotebookProvider); + serviceManager.add(IPlotViewer, PlotViewer); + serviceManager.addSingleton(IKernelLauncher, KernelLauncher); + serviceManager.addSingleton(IKernelFinder, KernelFinder); + serviceManager.addSingleton(ActiveEditorContextService, ActiveEditorContextService); + serviceManager.addSingleton(CellOutputMimeTypeTracker, CellOutputMimeTypeTracker, undefined, [IExtensionSingleActivationService, INotebookExecutionLogger]); + serviceManager.addSingleton(CommandRegistry, CommandRegistry); + serviceManager.addSingleton(DataViewerDependencyService, DataViewerDependencyService); + serviceManager.addSingleton(ICodeCssGenerator, CodeCssGenerator); + serviceManager.addSingleton(ICodeLensFactory, CodeLensFactory, undefined, [IInteractiveWindowListener]); + serviceManager.addSingleton(IDataScience, DataScience); + serviceManager.addSingleton(IDataScienceCodeLensProvider, DataScienceCodeLensProvider); + serviceManager.addSingleton(IDataScienceCommandListener, InteractiveWindowCommandListener); + serviceManager.addSingleton(IDataScienceCommandListener, NativeEditorCommandListener); + serviceManager.addSingleton(IDataViewerProvider, DataViewerProvider); + serviceManager.addSingleton(IDebugLocationTracker, DebugLocationTrackerFactory); + serviceManager.addSingleton(IExtensionSingleActivationService, Activation); + serviceManager.addSingleton(IExtensionSingleActivationService, Decorator); + serviceManager.addSingleton(IExtensionSingleActivationService, JupyterInterpreterSelectionCommand); + serviceManager.addSingleton(IExtensionSingleActivationService, PreWarmActivatedJupyterEnvironmentVariables); + serviceManager.addSingleton(IExtensionSingleActivationService, ServerPreload); + serviceManager.addSingleton(IInteractiveWindowListener, DataScienceSurveyBannerLogger); + serviceManager.addSingleton(IInteractiveWindowProvider, InteractiveWindowProvider); + serviceManager.addSingleton(IJupyterDebugger, JupyterDebugger, undefined, [ICellHashListener]); + serviceManager.addSingleton(IJupyterExecution, JupyterExecutionFactory); + serviceManager.addSingleton(IJupyterPasswordConnect, JupyterPasswordConnect); + serviceManager.addSingleton(IJupyterSessionManagerFactory, JupyterSessionManagerFactory); + serviceManager.addSingleton(IJupyterVariables, JupyterVariables); + serviceManager.addSingleton(INotebookEditorProvider, useCustomEditorApi ? NativeEditorProvider : NativeEditorProviderOld); + serviceManager.addSingleton(IPlotViewerProvider, PlotViewerProvider); + serviceManager.addSingleton(IStatusProvider, StatusProvider); + serviceManager.addSingleton(IThemeFinder, ThemeFinder); + serviceManager.addSingleton(JupyterCommandFinder, JupyterCommandFinder); + serviceManager.addSingleton(JupyterCommandLineSelector, JupyterCommandLineSelector); + serviceManager.addSingleton(JupyterCommandLineSelectorCommand, JupyterCommandLineSelectorCommand); + serviceManager.addSingleton(JupyterInterpreterDependencyService, JupyterInterpreterDependencyService); + serviceManager.addSingleton(JupyterInterpreterOldCacheStateStore, JupyterInterpreterOldCacheStateStore); + serviceManager.addSingleton(JupyterInterpreterSelector, JupyterInterpreterSelector); + serviceManager.addSingleton(JupyterInterpreterService, JupyterInterpreterService); + serviceManager.addSingleton(JupyterInterpreterStateStore, JupyterInterpreterStateStore); + serviceManager.addSingleton(JupyterServerSelector, JupyterServerSelector); + serviceManager.addSingleton(JupyterServerSelectorCommand, JupyterServerSelectorCommand); + serviceManager.addSingleton(KernelSelectionProvider, KernelSelectionProvider); + serviceManager.addSingleton(KernelSelector, KernelSelector); + serviceManager.addSingleton(KernelService, KernelService); + serviceManager.addSingleton(KernelSwitcher, KernelSwitcher); + serviceManager.addSingleton(KernelSwitcherCommand, KernelSwitcherCommand); + serviceManager.addSingleton(NotebookStarter, NotebookStarter); + serviceManager.addSingleton(ProgressReporter, ProgressReporter); + serviceManager.addSingleton(NativeEditorSynchronizer, NativeEditorSynchronizer); + serviceManager.addSingleton(INotebookProvider, NotebookProvider); + serviceManager.addSingleton(IJupyterServerProvider, NotebookServerProvider); + serviceManager.add(IJMPConnection, EnchannelJMPConnection); + serviceManager.addSingleton(IPyWidgetMessageDispatcherFactory, IPyWidgetMessageDispatcherFactory); + + // Temporary code, to allow users to revert to the old behavior. + const cfg = serviceManager.get(IWorkspaceService).getConfiguration('python.dataScience', undefined); + if (cfg.get('useOldJupyterServer', false)) { + serviceManager.addSingleton(IJupyterInterpreterDependencyManager, JupyterCommandInterpreterDependencyService); + serviceManager.addSingleton(IJupyterSubCommandExecutionService, JupyterCommandFinderInterpreterExecutionService); + } else { + serviceManager.addSingleton(IJupyterInterpreterDependencyManager, JupyterInterpreterSubCommandExecutionService); + serviceManager.addSingleton(IJupyterSubCommandExecutionService, JupyterInterpreterSubCommandExecutionService); + } + + registerGatherTypes(serviceManager); +} + +export function registerGatherTypes(serviceManager: IServiceManager) { + // tslint:disable-next-line: no-require-imports + const gather = require('./gather/gather'); + + serviceManager.add(IGatherProvider, gather.GatherProvider); + serviceManager.add(IGatherLogger, GatherLogger, undefined, [INotebookExecutionLogger]); +} diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index e419885d18c7..27f913c3d75b 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -139,6 +139,28 @@ export interface INotebookServer extends IAsyncDisposable { shutdown(): Promise; } +// Provides notebooks that talk directly to kernels as opposed to a jupyter server +export const IRawNotebookProvider = Symbol('IRawNotebookProvider'); +export interface IRawNotebookProvider extends IAsyncDisposable { + connect(): Promise; + createNotebook( + identity: Uri, + resource: Resource, + notebookMetadata?: nbformat.INotebookMetadata, + cancelToken?: CancellationToken + ): Promise; + getNotebook(identity: Uri): Promise; +} + +// Provides notebooks that talk to jupyter servers +export const IJupyterNotebookProvider = Symbol('IJupyterNotebookProvider'); +export interface IJupyterNotebookProvider { + connect(options: ConnectNotebookProviderOptions): Promise; + createNotebook(options: GetNotebookOptions): Promise; + getNotebook(options: GetNotebookOptions): Promise; + disconnect(options: ConnectNotebookProviderOptions): Promise; +} + export interface INotebook extends IAsyncDisposable { readonly resource: Resource; readonly connection: INotebookProviderConnection | undefined; @@ -1004,8 +1026,8 @@ export interface INotebookProvider { disconnect(options: ConnectNotebookProviderOptions): Promise; } -export const INotebookServerProvider = Symbol('INotebookServerProvider'); -export interface INotebookServerProvider { +export const IJupyterServerProvider = Symbol('IJupyterServerProvider'); +export interface IJupyterServerProvider { /** * Gets the server used for starting notebooks */ diff --git a/src/test/datascience/dataScienceIocContainer.ts b/src/test/datascience/dataScienceIocContainer.ts index 0520dc4ed33b..769179446b1d 100644 --- a/src/test/datascience/dataScienceIocContainer.ts +++ b/src/test/datascience/dataScienceIocContainer.ts @@ -210,6 +210,7 @@ import { JupyterDebugger } from '../../client/datascience/jupyter/jupyterDebugge import { JupyterExecutionFactory } from '../../client/datascience/jupyter/jupyterExecutionFactory'; import { JupyterExporter } from '../../client/datascience/jupyter/jupyterExporter'; import { JupyterImporter } from '../../client/datascience/jupyter/jupyterImporter'; +import { JupyterNotebookProvider } from '../../client/datascience/jupyter/jupyterNotebookProvider'; import { JupyterPasswordConnect } from '../../client/datascience/jupyter/jupyterPasswordConnect'; import { JupyterServerWrapper } from '../../client/datascience/jupyter/jupyterServerWrapper'; import { JupyterSessionManagerFactory } from '../../client/datascience/jupyter/jupyterSessionManagerFactory'; @@ -228,6 +229,7 @@ import { PlotViewer } from '../../client/datascience/plotting/plotViewer'; import { PlotViewerProvider } from '../../client/datascience/plotting/plotViewerProvider'; import { ProgressReporter } from '../../client/datascience/progress/progressReporter'; import { EnchannelJMPConnection } from '../../client/datascience/raw-kernel/enchannelJMPConnection'; +import { RawNotebookProviderWrapper } from '../../client/datascience/raw-kernel/rawNotebookProviderWrapper'; import { StatusProvider } from '../../client/datascience/statusProvider'; import { ThemeFinder } from '../../client/datascience/themeFinder'; import { @@ -253,7 +255,9 @@ import { IJupyterDebugger, IJupyterExecution, IJupyterInterpreterDependencyManager, + IJupyterNotebookProvider, IJupyterPasswordConnect, + IJupyterServerProvider, IJupyterSessionManagerFactory, IJupyterSubCommandExecutionService, IJupyterVariables, @@ -264,10 +268,10 @@ import { INotebookImporter, INotebookProvider, INotebookServer, - INotebookServerProvider, INotebookStorage, IPlotViewer, IPlotViewerProvider, + IRawNotebookProvider, IStatusProvider, IThemeFinder } from '../../client/datascience/types'; @@ -568,6 +572,7 @@ export class DataScienceIocContainer extends UnitTestIocContainer { this.serviceManager.addSingleton(IExtensions, MockExtensions); this.serviceManager.add(INotebookServer, JupyterServerWrapper); this.serviceManager.add(IJupyterCommandFactory, JupyterCommandFactory); + this.serviceManager.addSingleton(IRawNotebookProvider, RawNotebookProviderWrapper); this.serviceManager.addSingleton(IThemeFinder, ThemeFinder); this.serviceManager.addSingleton(ICodeCssGenerator, CodeCssGenerator); this.serviceManager.addSingleton(IStatusProvider, StatusProvider); @@ -703,7 +708,8 @@ export class DataScienceIocContainer extends UnitTestIocContainer { } this.serviceManager.addSingleton(INotebookProvider, NotebookProvider); - this.serviceManager.addSingleton(INotebookServerProvider, NotebookServerProvider); + this.serviceManager.addSingleton(IJupyterNotebookProvider, JupyterNotebookProvider); + this.serviceManager.addSingleton(IJupyterServerProvider, NotebookServerProvider); this.serviceManager.add(IInteractiveWindowListener, IntellisenseProvider); this.serviceManager.add(IInteractiveWindowListener, AutoSaveService); diff --git a/src/test/datascience/interactive-common/notebookProvider.unit.test.ts b/src/test/datascience/interactive-common/notebookProvider.unit.test.ts index a40070012a52..b350bdc198c9 100644 --- a/src/test/datascience/interactive-common/notebookProvider.unit.test.ts +++ b/src/test/datascience/interactive-common/notebookProvider.unit.test.ts @@ -5,14 +5,20 @@ import { anything, instance, mock, when } from 'ts-mockito'; import * as typemoq from 'typemoq'; import * as vscode from 'vscode'; import { IFileSystem } from '../../../client/common/platform/types'; -import { IDisposableRegistry } from '../../../client/common/types'; +import { + IConfigurationService, + IDataScienceSettings, + IDisposableRegistry, + IExperimentsManager, + IPythonSettings +} from '../../../client/common/types'; import { NotebookProvider } from '../../../client/datascience/interactive-common/notebookProvider'; import { IInteractiveWindowProvider, + IJupyterNotebookProvider, INotebook, INotebookEditorProvider, - INotebookServer, - INotebookServerProvider + IRawNotebookProvider } from '../../../client/datascience/types'; function Uri(filename: string): vscode.Uri { @@ -36,90 +42,74 @@ suite('Data Science - NotebookProvider', () => { let notebookEditorProvider: INotebookEditorProvider; let interactiveWindowProvider: IInteractiveWindowProvider; let disposableRegistry: IDisposableRegistry; - let notebookServerProvider: INotebookServerProvider; + let jupyterNotebookProvider: IJupyterNotebookProvider; + let rawNotebookProvider: IRawNotebookProvider; + let experimentsManager: IExperimentsManager; + let configuration: IConfigurationService; + let pythonSettings: IPythonSettings; + let dataScienceSettings: IDataScienceSettings; setup(() => { fileSystem = mock(); notebookEditorProvider = mock(); interactiveWindowProvider = mock(); disposableRegistry = mock(); - notebookServerProvider = mock(); + jupyterNotebookProvider = mock(); + rawNotebookProvider = mock(); + experimentsManager = mock(); + configuration = mock(); + + // Set up our settings + pythonSettings = mock(); + dataScienceSettings = mock(); + when(pythonSettings.datascience).thenReturn(instance(dataScienceSettings)); + when(dataScienceSettings.jupyterServerURI).thenReturn('local'); + when(dataScienceSettings.useDefaultConfigForJupyter).thenReturn(true); + when(configuration.getSettings(anything())).thenReturn(instance(pythonSettings)); + + // Set up experiment manager + when(experimentsManager.inExperiment(anything())).thenReturn(false); + notebookProvider = new NotebookProvider( instance(fileSystem), instance(notebookEditorProvider), instance(interactiveWindowProvider), instance(disposableRegistry), - instance(notebookServerProvider) + instance(rawNotebookProvider), + instance(jupyterNotebookProvider), + instance(configuration), + instance(experimentsManager) ); }); - test('NotebookProvider getOrCreateNotebook no server', async () => { - when(notebookServerProvider.getOrCreateServer(anything())).thenResolve(undefined); - - const notebook = await notebookProvider.getOrCreateNotebook({ identity: Uri('C:\\\\foo.py') }); - expect(notebook).to.equal(undefined, 'No server should return no notebook'); - }); - - test('NotebookProvider getOrCreateNotebook server has notebook already', async () => { - const notebookServer = createTypeMoq('jupyter server'); - const notebookMock = createTypeMoq('jupyter notebook'); - notebookServer - .setup(s => s.getNotebook(typemoq.It.isAny())) - .returns(() => Promise.resolve(notebookMock.object)); - when(notebookServerProvider.getOrCreateServer(anything())).thenResolve(notebookServer.object); - - const notebook = await notebookProvider.getOrCreateNotebook({ identity: Uri('C:\\\\foo.py') }); - expect(notebook).to.not.equal(undefined, 'Server should return a notebook'); - }); - - test('NotebookProvider getOrCreateNotebook server does not have notebook already', async () => { - const notebookServer = createTypeMoq('jupyter server'); + test('NotebookProvider getOrCreateNotebook jupyter provider has notebook already', async () => { const notebookMock = createTypeMoq('jupyter notebook'); - // Get notebook undefined, but create notebook set - notebookServer.setup(s => s.getNotebook(typemoq.It.isAny())).returns(() => Promise.resolve(undefined)); - notebookServer - .setup(s => s.createNotebook(typemoq.It.isAny(), typemoq.It.isAny(), typemoq.It.isAny())) - .returns(() => Promise.resolve(notebookMock.object)); - when(notebookServerProvider.getOrCreateServer(anything())).thenResolve(notebookServer.object); + when(jupyterNotebookProvider.getNotebook(anything())).thenResolve(notebookMock.object); const notebook = await notebookProvider.getOrCreateNotebook({ identity: Uri('C:\\\\foo.py') }); - expect(notebook).to.not.equal(undefined, 'Server should return a notebook'); + expect(notebook).to.not.equal(undefined, 'Provider should return a notebook'); }); - test('NotebookProvider getOrCreateNotebook getOnly server does not have notebook already', async () => { - const notebookServer = createTypeMoq('jupyter server'); + test('NotebookProvider getOrCreateNotebook jupyter provider does not have notebook already', async () => { const notebookMock = createTypeMoq('jupyter notebook'); - // Get notebook undefined, but create notebook set - notebookServer.setup(s => s.getNotebook(typemoq.It.isAny())).returns(() => Promise.resolve(undefined)); - notebookServer - .setup(s => s.createNotebook(typemoq.It.isAny(), typemoq.It.isAny(), typemoq.It.isAny())) - .returns(() => Promise.resolve(notebookMock.object)); - when(notebookServerProvider.getOrCreateServer(anything())).thenResolve(notebookServer.object); + when(jupyterNotebookProvider.getNotebook(anything())).thenResolve(undefined); + when(jupyterNotebookProvider.createNotebook(anything())).thenResolve(notebookMock.object); + when(jupyterNotebookProvider.connect(anything())).thenResolve({} as any); const notebook = await notebookProvider.getOrCreateNotebook({ identity: Uri('C:\\\\foo.py') }); - expect(notebook).to.not.equal(undefined, 'Server should return a notebook'); + expect(notebook).to.not.equal(undefined, 'Provider should return a notebook'); }); test('NotebookProvider getOrCreateNotebook second request should return the notebook already cached', async () => { - const notebookServer = createTypeMoq('jupyter server'); const notebookMock = createTypeMoq('jupyter notebook'); - // Get notebook undefined, but create notebook set - notebookServer.setup(s => s.getNotebook(typemoq.It.isAny())).returns(() => Promise.resolve(undefined)); - notebookServer - .setup(s => s.createNotebook(typemoq.It.isAny(), typemoq.It.isAny(), typemoq.It.isAny())) - .returns(() => Promise.resolve(notebookMock.object)); - when(notebookServerProvider.getOrCreateServer(anything())).thenResolve(notebookServer.object); + when(jupyterNotebookProvider.getNotebook(anything())).thenResolve(undefined); + when(jupyterNotebookProvider.createNotebook(anything())).thenResolve(notebookMock.object); + when(jupyterNotebookProvider.connect(anything())).thenResolve({} as any); const notebook = await notebookProvider.getOrCreateNotebook({ identity: Uri('C:\\\\foo.py') }); expect(notebook).to.not.equal(undefined, 'Server should return a notebook'); const notebook2 = await notebookProvider.getOrCreateNotebook({ identity: Uri('C:\\\\foo.py') }); expect(notebook2).to.equal(notebook); - - // Only one create call - notebookServer.verify( - s => s.createNotebook(typemoq.It.isAny(), typemoq.It.isAny(), typemoq.It.isAny()), - typemoq.Times.once() - ); }); }); From 13a9f34721d10024d54180bfdf693cbc3db8f6d5 Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Tue, 14 Apr 2020 08:10:44 -0700 Subject: [PATCH 015/725] Merge Kernel launcher with existing raw kernel work (#11131) --- package.nls.json | 2 + src/client/common/utils/localize.ts | 8 ++ .../jupyter/liveshare/serverCache.ts | 43 +----- .../kernel-launcher/kernelLauncher.ts | 123 +++++++++++++----- .../datascience/kernel-launcher/types.ts | 8 +- .../liveshare/hostRawNotebookProvider.ts | 70 +++++----- .../raw-kernel/rawJupyterSession.ts | 110 ++++++++++++++-- .../datascience/raw-kernel/rawKernel.ts | 13 +- .../raw-kernel/rawNotebookProviderWrapper.ts | 10 +- .../datascience/raw-kernel/rawSession.ts | 12 +- src/client/datascience/utils.ts | 49 +++++++ .../raw-kernel/rawJupyterSession.unit.test.ts | 80 ++++++++---- .../raw-kernel/rawKernel.unit.test.ts | 35 +---- .../raw-kernel/rawSession.unit.test.ts | 36 +---- 14 files changed, 366 insertions(+), 233 deletions(-) create mode 100644 src/client/datascience/utils.ts diff --git a/package.nls.json b/package.nls.json index 9da4caf938dc..d8c031a6c300 100644 --- a/package.nls.json +++ b/package.nls.json @@ -97,6 +97,8 @@ "DataScience.dataExplorerTitle": "Data Viewer", "DataScience.badWebPanelFormatString": "

{0} is not a valid file name

", "DataScience.sessionDisposed": "Cannot execute code, session has been disposed.", + "DataScience.rawKernelProcessNotStarted": "Raw kernel process was not able to start.", + "DataScience.rawKernelProcessExitBeforeConnect": "Raw kernel process exited before connecting.", "DataScience.passwordFailure": "Failed to connect to password protected server. Check that password is correct.", "DataScience.exportDialogTitle": "Export to Jupyter Notebook", "DataScience.exportDialogFilter": "Jupyter Notebooks", diff --git a/src/client/common/utils/localize.ts b/src/client/common/utils/localize.ts index bd7ad53ca41e..ed4c4d03cec1 100644 --- a/src/client/common/utils/localize.ts +++ b/src/client/common/utils/localize.ts @@ -266,6 +266,14 @@ export namespace DataScience { 'DataScience.passwordFailure', 'Failed to connect to password protected server. Check that password is correct.' ); + export const rawKernelProcessNotStarted = localize( + 'DataScience.rawKernelProcessNotStarted', + 'Raw kernel process was not able to start.' + ); + export const rawKernelProcessExitBeforeConnect = localize( + 'DataScience.rawKernelProcessExitBeforeConnect', + 'Raw kernel process exited before connecting.' + ); export const unknownMimeTypeFormat = localize( 'DataScience.unknownMimeTypeFormat', 'Mime type {0} is not currently supported' diff --git a/src/client/datascience/jupyter/liveshare/serverCache.ts b/src/client/datascience/jupyter/liveshare/serverCache.ts index 6ef3fe5a1a97..2f5470dc4ab4 100644 --- a/src/client/datascience/jupyter/liveshare/serverCache.ts +++ b/src/client/datascience/jupyter/liveshare/serverCache.ts @@ -3,7 +3,6 @@ 'use strict'; import '../../../common/extensions'; -import * as path from 'path'; import * as uuid from 'uuid/v4'; import { CancellationToken, CancellationTokenSource } from 'vscode'; @@ -11,6 +10,7 @@ import { IWorkspaceService } from '../../../common/application/types'; import { IFileSystem } from '../../../common/platform/types'; import { IAsyncDisposable, IConfigurationService } from '../../../common/types'; import { INotebookServer, INotebookServerOptions } from '../../types'; +import { calculateWorkingDirectory } from '../../utils'; interface IServerData { options: INotebookServerOptions; @@ -111,7 +111,10 @@ export class ServerCache implements IAsyncDisposable { skipUsingDefaultConfig: options ? options.skipUsingDefaultConfig : false, // Default for this is false usingDarkTheme: options ? options.usingDarkTheme : undefined, purpose: options ? options.purpose : uuid(), - workingDir: options && options.workingDir ? options.workingDir : await this.calculateWorkingDirectory(), + workingDir: + options && options.workingDir + ? options.workingDir + : await calculateWorkingDirectory(this.configService, this.workspace, this.fileSystem), metadata: options?.metadata, allowUI: options?.allowUI ? options.allowUI : () => false }; @@ -127,40 +130,4 @@ export class ServerCache implements IAsyncDisposable { return `${options.purpose}${uri}${useFlag}${options.workingDir}`; } } - - private async calculateWorkingDirectory(): Promise { - let workingDir: string | undefined; - // For a local launch calculate the working directory that we should switch into - const settings = this.configService.getSettings(undefined); - const fileRoot = settings.datascience.notebookFileRoot; - - // If we don't have a workspace open the notebookFileRoot seems to often have a random location in it (we use ${workspaceRoot} as default) - // so only do this setting if we actually have a valid workspace open - if (fileRoot && this.workspace.hasWorkspaceFolders) { - const workspaceFolderPath = this.workspace.workspaceFolders![0].uri.fsPath; - if (path.isAbsolute(fileRoot)) { - if (await this.fileSystem.directoryExists(fileRoot)) { - // User setting is absolute and exists, use it - workingDir = fileRoot; - } else { - // User setting is absolute and doesn't exist, use workspace - workingDir = workspaceFolderPath; - } - } else if (!fileRoot.includes('${')) { - // fileRoot is a relative path, combine it with the workspace folder - const combinedPath = path.join(workspaceFolderPath, fileRoot); - if (await this.fileSystem.directoryExists(combinedPath)) { - // combined path exists, use it - workingDir = combinedPath; - } else { - // Combined path doesn't exist, use workspace - workingDir = workspaceFolderPath; - } - } else { - // fileRoot is a variable that hasn't been expanded - workingDir = fileRoot; - } - } - return workingDir; - } } diff --git a/src/client/datascience/kernel-launcher/kernelLauncher.ts b/src/client/datascience/kernel-launcher/kernelLauncher.ts index 1206396ec200..ce0e9426a2af 100644 --- a/src/client/datascience/kernel-launcher/kernelLauncher.ts +++ b/src/client/datascience/kernel-launcher/kernelLauncher.ts @@ -7,9 +7,13 @@ import { inject, injectable } from 'inversify'; import * as portfinder from 'portfinder'; import { promisify } from 'util'; import * as uuid from 'uuid/v4'; +import { Event, EventEmitter } from 'vscode'; import { InterpreterUri } from '../../common/installer/types'; +import { traceInfo, traceWarning } from '../../common/logger'; import { IFileSystem, TemporaryFile } from '../../common/platform/types'; import { IPythonExecutionFactory } from '../../common/process/types'; +import { createDeferred, Deferred } from '../../common/utils/async'; +import * as localize from '../../common/utils/localize'; import { isResource, noop } from '../../common/utils/misc'; import { IJupyterKernelSpec } from '../types'; import { IKernelConnection, IKernelFinder, IKernelLauncher, IKernelProcess } from './types'; @@ -18,28 +22,50 @@ import { IKernelConnection, IKernelFinder, IKernelLauncher, IKernelProcess } fro // Exposes connection information and the process itself. class KernelProcess implements IKernelProcess { private _process?: ChildProcess; - private _connection?: IKernelConnection; + //private _connection: IKernelConnection; + //private _kernelSpec: IJupyterKernelSpec; + //private _interpreterUri: InterpreterUri; private connectionFile?: TemporaryFile; + private readyPromise: Deferred; + private exitEvent: EventEmitter = new EventEmitter(); + + // This promise is resolved when the launched process is ready to get JMP messages + public get ready(): Promise { + return this.readyPromise.promise; + } + + // This event is triggered if the process is exited + public get exited(): Event { + return this.exitEvent.event; + } + + public get kernelSpec(): Readonly { + return this._kernelSpec; + } public get process(): ChildProcess | undefined { return this._process; } - public get connection(): IKernelConnection | undefined { + public get connection(): Readonly { return this._connection; } constructor( - @inject(IPythonExecutionFactory) private executionFactory: IPythonExecutionFactory, - @inject(IFileSystem) private file: IFileSystem - ) {} + private executionFactory: IPythonExecutionFactory, + private file: IFileSystem, + private _connection: IKernelConnection, + private _kernelSpec: IJupyterKernelSpec, + private _interpreter: InterpreterUri + ) { + this.readyPromise = createDeferred(); + } - public async launch(interpreter: InterpreterUri, kernelSpec: IJupyterKernelSpec): Promise { - this.connectionFile = await this.file.createTemporaryFile('json'); + public async launch(): Promise { + this.connectionFile = await this.file.createTemporaryFile('.json'); - const resource = isResource(interpreter) ? interpreter : undefined; - const pythonPath = isResource(interpreter) ? undefined : interpreter.path; + const resource = isResource(this._interpreter) ? this._interpreter : undefined; + const pythonPath = isResource(this._interpreter) ? undefined : this._interpreter.path; - const args = [...kernelSpec.argv]; - this._connection = await this.getKernelConnection(); + const args = [...this._kernelSpec.argv]; await this.file.writeFile(this.connectionFile.filePath, JSON.stringify(this._connection), { encoding: 'utf-8', flag: 'w' @@ -50,7 +76,35 @@ class KernelProcess implements IKernelProcess { args.splice(0, 1); const executionService = await this.executionFactory.create({ resource, pythonPath }); - this._process = executionService.execObservable(args, {}).proc; + const exeObs = executionService.execObservable(args, {}); + + if (exeObs.proc) { + exeObs.proc!.on('exit', exitCode => { + traceInfo('KernelProcess Exit', `Exit - ${exitCode}`); + if (!this.readyPromise.completed) { + this.readyPromise.reject(new Error(localize.DataScience.rawKernelProcessExitBeforeConnect())); + } + this.exitEvent.fire(exitCode); + }); + } else { + traceInfo('KernelProcess failed to launch'); + this.readyPromise.reject(new Error(localize.DataScience.rawKernelProcessNotStarted())); + } + exeObs.out.subscribe(output => { + if (output.source === 'stderr') { + traceWarning(`StdErr from Kernel Process ${output.out}`); + } else { + // Search for --existing this is the message that will indicate that our kernel is actually + // up and started from stdout + // To connect another client to this kernel, use: + // --existing /var/folders/q7/cn8fg6s94fgdcl0h7rbxldf00000gn/T/tmp-16231TOL2dgBoWET1.json + if (!this.readyPromise.completed && output.out.includes('--existing')) { + this.readyPromise.resolve(); + } + traceInfo(output.out); + } + }); + this._process = exeObs.proc; } public dispose() { @@ -61,6 +115,32 @@ class KernelProcess implements IKernelProcess { noop(); } } +} + +// Launches and returns a kernel process given a resource or python interpreter. +// If the given interpreter is undefined, it will try to use the selected interpreter. +// If the selected interpreter doesn't have a kernel, it will find a kernel on disk and use that. +@injectable() +export class KernelLauncher implements IKernelLauncher { + constructor( + @inject(IKernelFinder) private kernelFinder: IKernelFinder, + @inject(IPythonExecutionFactory) private executionFactory: IPythonExecutionFactory, + @inject(IFileSystem) private file: IFileSystem + ) {} + + public async launch(interpreterUri: InterpreterUri, kernelName?: string): Promise { + const kernelSpec = await this.kernelFinder.findKernelSpec(interpreterUri, kernelName); + const connection = await this.getKernelConnection(); + const kernelProcess = new KernelProcess( + this.executionFactory, + this.file, + connection, + kernelSpec, + interpreterUri + ); + await kernelProcess.launch(); + return kernelProcess; + } private async getKernelConnection(): Promise { const getPorts = promisify(portfinder.getPorts); @@ -80,22 +160,3 @@ class KernelProcess implements IKernelProcess { }; } } - -// Launches and returns a kernel process given a resource or python interpreter. -// If the given interpreter is undefined, it will try to use the selected interpreter. -// If the selected interpreter doesn't have a kernel, it will find a kernel on disk and use that. -@injectable() -export class KernelLauncher implements IKernelLauncher { - constructor( - @inject(IKernelFinder) private kernelFinder: IKernelFinder, - @inject(IPythonExecutionFactory) private executionFactory: IPythonExecutionFactory, - @inject(IFileSystem) private file: IFileSystem - ) {} - - public async launch(interpreterUri: InterpreterUri, kernelName?: string): Promise { - const kernelSpec = await this.kernelFinder.findKernelSpec(interpreterUri, kernelName); - const kernelProcess = new KernelProcess(this.executionFactory, this.file); - await kernelProcess.launch(interpreterUri, kernelSpec); - return kernelProcess; - } -} diff --git a/src/client/datascience/kernel-launcher/types.ts b/src/client/datascience/kernel-launcher/types.ts index c554bb693b60..db657a34f7bb 100644 --- a/src/client/datascience/kernel-launcher/types.ts +++ b/src/client/datascience/kernel-launcher/types.ts @@ -4,12 +4,13 @@ import { ChildProcess } from 'child_process'; import { IDisposable } from 'monaco-editor'; +import { Event } from 'vscode'; import { InterpreterUri } from '../../common/installer/types'; import { IJupyterKernelSpec } from '../types'; export const IKernelLauncher = Symbol('IKernelLauncher'); export interface IKernelLauncher { - launch(interpreterUri: InterpreterUri, kernelName: string): Promise; + launch(interpreterUri: InterpreterUri, kernelName?: string): Promise; } export interface IKernelConnection { @@ -27,7 +28,10 @@ export interface IKernelConnection { export interface IKernelProcess extends IDisposable { process: ChildProcess | undefined; - connection: IKernelConnection | undefined; + readonly connection: Readonly | undefined; + ready: Promise; + readonly kernelSpec: Readonly | undefined; + exited: Event; dispose(): void; launch(interpreter: InterpreterUri, kernelSpec: IJupyterKernelSpec): Promise; } diff --git a/src/client/datascience/raw-kernel/liveshare/hostRawNotebookProvider.ts b/src/client/datascience/raw-kernel/liveshare/hostRawNotebookProvider.ts index dcdecff882be..f2e7f2a9d26b 100644 --- a/src/client/datascience/raw-kernel/liveshare/hostRawNotebookProvider.ts +++ b/src/client/datascience/raw-kernel/liveshare/hostRawNotebookProvider.ts @@ -9,7 +9,7 @@ import * as vsls from 'vsls/vscode'; import { nbformat } from '@jupyterlab/coreutils'; import { IApplicationShell, ILiveShareApi, IWorkspaceService } from '../../../common/application/types'; -import { traceInfo } from '../../../common/logger'; +import { traceError, traceInfo } from '../../../common/logger'; import { IFileSystem } from '../../../common/platform/types'; import { IAsyncDisposableRegistry, IConfigurationService, IDisposableRegistry, Resource } from '../../../common/types'; import { createDeferred } from '../../../common/utils/async'; @@ -18,8 +18,15 @@ import { Identifiers, LiveShare, Settings } from '../../constants'; import { HostJupyterNotebook } from '../../jupyter/liveshare/hostJupyterNotebook'; import { LiveShareParticipantHost } from '../../jupyter/liveshare/liveShareParticipantMixin'; import { IRoleBasedObject } from '../../jupyter/liveshare/roleBasedFactory'; -import { INotebook, INotebookExecutionInfo, INotebookExecutionLogger, IRawNotebookProvider } from '../../types'; -import { EnchannelJMPConnection } from '../enchannelJMPConnection'; +import { IKernelLauncher } from '../../kernel-launcher/types'; +import { + IJupyterKernelSpec, + INotebook, + INotebookExecutionInfo, + INotebookExecutionLogger, + IRawNotebookProvider +} from '../../types'; +import { calculateWorkingDirectory } from '../../utils'; import { RawJupyterSession } from '../rawJupyterSession'; import { RawNotebookProviderBase } from '../rawNotebookProvider'; @@ -38,7 +45,8 @@ export class HostRawNotebookProvider private workspaceService: IWorkspaceService, private appShell: IApplicationShell, private fs: IFileSystem, - private serviceContainer: IServiceContainer + private serviceContainer: IServiceContainer, + private kernelLauncher: IKernelLauncher ) { super(liveShare, asyncRegistry); } @@ -72,35 +80,21 @@ export class HostRawNotebookProvider notebookMetadata?: nbformat.INotebookMetadata, cancelToken?: CancellationToken ): Promise { - throw new Error('Not implemented'); - // RAWKERNEL: Hack to create session, uncomment throw and update ci to connect to a running kernel - const ci = { - version: 0, - transport: 'tcp', - ip: '127.0.0.1', - shell_port: 51065, - iopub_port: 51066, - stdin_port: 51067, - hb_port: 51069, - control_port: 51068, - signature_scheme: 'hmac-sha256', - key: '9a4f68cd-b5e4887e4b237ea4c91c265c' - }; - const rawSession = new RawJupyterSession(new EnchannelJMPConnection()); - try { - await rawSession.connect(ci); - } finally { - if (!rawSession.isConnected) { - await rawSession.dispose(); - } - } - const notebookPromise = createDeferred(); this.setNotebook(identity, notebookPromise.promise); + const rawSession = new RawJupyterSession(this.kernelLauncher, this.serviceContainer); try { + const launchTimeout = this.configService.getSettings().datascience.jupyterLaunchTimeout; + const launchedKernelSpec = await rawSession.connect( + resource, + launchTimeout, + notebookMetadata?.kernelspec?.name, + cancelToken + ); + // Get the execution info for our notebook - const info = this.getExecutionInfo(resource, notebookMetadata); + const info = await this.getExecutionInfo(launchedKernelSpec); if (rawSession.isConnected) { // Create our notebook @@ -119,11 +113,6 @@ export class HostRawNotebookProvider this.fs ); - // Wait for it to be ready - traceInfo(`Waiting for idle (session) ${this.id}`); - const idleTimeout = this.configService.getSettings().datascience.jupyterLaunchTimeout; - await notebook.waitForIdle(idleTimeout); - // Run initial setup await notebook.initialize(cancelToken); @@ -134,6 +123,10 @@ export class HostRawNotebookProvider notebookPromise.reject(this.getDisposedError()); } } catch (ex) { + // Make sure we shut down our session in case we started a process + rawSession.dispose().catch(error => { + traceError(`Failed to dispose of raw session on launch error: ${error} `); + }); // If there's an error, then reject the promise that is returned. // This original promise must be rejected as it is cached (check `setNotebook`). notebookPromise.reject(ex); @@ -142,17 +135,14 @@ export class HostRawNotebookProvider return notebookPromise.promise; } - // RAWKERNEL: Not the real execution info, just stub it out for now - private getExecutionInfo( - _resource: Resource, - _notebookMetadata?: nbformat.INotebookMetadata - ): INotebookExecutionInfo { + // Get the notebook execution info for this raw session instance + private async getExecutionInfo(kernelSpec?: IJupyterKernelSpec): Promise { return { connectionInfo: this.getConnection(), uri: Settings.JupyterServerLocalLaunch, interpreter: undefined, - kernelSpec: undefined, - workingDir: undefined, + kernelSpec: kernelSpec, + workingDir: await calculateWorkingDirectory(this.configService, this.workspaceService, this.fs), purpose: Identifiers.RawPurpose }; } diff --git a/src/client/datascience/raw-kernel/rawJupyterSession.ts b/src/client/datascience/raw-kernel/rawJupyterSession.ts index 570c8b8e5cc1..bfaba6f5bff7 100644 --- a/src/client/datascience/raw-kernel/rawJupyterSession.ts +++ b/src/client/datascience/raw-kernel/rawJupyterSession.ts @@ -2,28 +2,36 @@ // Licensed under the MIT License. 'use strict'; import { CancellationToken } from 'vscode-jsonrpc'; -import { traceInfo } from '../../common/logger'; +import { CancellationError, createPromiseFromCancellation } from '../../common/cancellation'; +import { traceError, traceInfo } from '../../common/logger'; +import { IDisposable, Resource } from '../../common/types'; +import { waitForPromise } from '../../common/utils/async'; +import * as localize from '../../common/utils/localize'; +import { IServiceContainer } from '../../ioc/types'; import { BaseJupyterSession } from '../baseJupyterSession'; import { LiveKernelModel } from '../jupyter/kernels/types'; +import { IKernelConnection, IKernelLauncher, IKernelProcess } from '../kernel-launcher/types'; import { reportAction } from '../progress/decorator'; import { ReportableAction } from '../progress/types'; import { RawSession } from '../raw-kernel/rawSession'; -import { IJMPConnection, IJMPConnectionInfo, IJupyterKernelSpec } from '../types'; +import { IJMPConnection, IJupyterKernelSpec } from '../types'; /* -RawJupyterSession is the implementation of IJupyterSession that instead off +RawJupyterSession is the implementation of IJupyterSession that instead of connecting to JupyterLab services it instead connects to a kernel directly through ZMQ. It's responsible for translating our IJupyterSession interface into the jupyterlabs interface as well as starting up and connecting to a raw session */ export class RawJupyterSession extends BaseJupyterSession { - private rawSession: RawSession; + private currentKernelProcess: IKernelProcess | undefined; + private processExitHandler: IDisposable | undefined; - constructor(connection: IJMPConnection) { + constructor( + private readonly kernelLauncher: IKernelLauncher, + private readonly serviceContainer: IServiceContainer + ) { super(); - this.rawSession = new RawSession(connection); - this.session = this.rawSession; } public async shutdown(): Promise { @@ -32,6 +40,14 @@ export class RawJupyterSession extends BaseJupyterSession { this.session = undefined; } + // Unhook our process exit handler before we dispose the process ourselves + this.processExitHandler?.dispose(); // NOSONAR + this.processExitHandler = undefined; + + if (this.currentKernelProcess) { + this.currentKernelProcess.dispose(); + } + if (this.onStatusChangedEvent) { this.onStatusChangedEvent.dispose(); } @@ -47,15 +63,87 @@ export class RawJupyterSession extends BaseJupyterSession { throw new Error('Not implemented'); } - // RAWKERNEL: Cancel token routed down? - public async connect(connectionInfo: IJMPConnectionInfo, _cancelToken?: CancellationToken): Promise { - await this.rawSession.connect(connectionInfo); + public async connect( + resource: Resource, + timeout: number, + kernelName?: string, + cancelToken?: CancellationToken + ): Promise { + try { + // Try to start up our raw session, allow for cancellation or timeout + // Notebook Provider level will handle the thrown error + const rawSessionStart = await waitForPromise( + Promise.race([ + this.startRawSession(resource, kernelName), + createPromiseFromCancellation({ + cancelAction: 'reject', + defaultValue: new CancellationError(), + token: cancelToken + }) + ]), + timeout + ); + + // Only connect our session if we didn't cancel or timeout + if (rawSessionStart instanceof CancellationError) { + traceInfo('Starting of raw session cancelled by user'); + throw rawSessionStart; + } else if (rawSessionStart === null) { + traceError('Raw session failed to start in given timeout'); + throw new Error(localize.DataScience.sessionDisposed()); + } else { + traceInfo('Raw session started and connected'); + this.session = rawSessionStart.session; + this.currentKernelProcess = rawSessionStart.process; + } + } catch (error) { + traceError(`Failed to connect raw kernel session: ${error}`); + this.connected = false; + throw error; + } - // At this point we are connected and ready to work this.connected = true; + return this.currentKernelProcess.kernelSpec; } public async changeKernel(_kernel: IJupyterKernelSpec | LiveKernelModel, _timeoutMS: number): Promise { throw new Error('Not implemented'); } + + private async startRawSession( + resource: Resource, + kernelName?: string + ): Promise<{ session: RawSession; process: IKernelProcess }> { + const process = await this.kernelLauncher.launch(resource, kernelName); + + if (!process.connection) { + traceError('KernelProcess launched without connection info'); + throw new Error(localize.DataScience.sessionDisposed()); + } + + // Watch to see if our process exits + this.processExitHandler = process.exited(exitCode => { + traceError(`Raw kernel process exited code: ${exitCode}`); + this.shutdown().catch(reason => { + traceError(`Error shutting down raw jupyter session: ${reason}`); + }); + // Next code the user executes will show a session disposed message + }); + + // Wait for the process to actually be ready to connect to + await process.ready; + + const connection = await this.jmpConnection(process.connection); + const session = new RawSession(connection); + return { session, process }; + } + + // Create and connect our JMP (Jupyter Messaging Protocol) for talking to the raw kernel + private async jmpConnection(kernelConnection: IKernelConnection): Promise { + const connection = this.serviceContainer.get(IJMPConnection); + + await connection.connect(kernelConnection); + + return connection; + } } diff --git a/src/client/datascience/raw-kernel/rawKernel.ts b/src/client/datascience/raw-kernel/rawKernel.ts index b842049474ab..661b19db30c2 100644 --- a/src/client/datascience/raw-kernel/rawKernel.ts +++ b/src/client/datascience/raw-kernel/rawKernel.ts @@ -7,7 +7,7 @@ import { ISignal, Signal } from '@phosphor/signaling'; import cloneDeep = require('lodash/cloneDeep'); import * as uuid from 'uuid/v4'; import { traceError } from '../../common/logger'; -import { IJMPConnection, IJMPConnectionInfo } from '../types'; +import { IJMPConnection } from '../types'; import { RawFuture } from './rawFuture'; /* @@ -89,18 +89,15 @@ export class RawKernel implements Kernel.IKernel { RawFuture >(); - // JMP connection should be injected, but no need to yet until it actually exists - constructor(jmpConnection: IJMPConnection, clientID: string) { + constructor(jmpConnection: IJMPConnection, clientId: string) { // clientID is controlled by the session as we keep the same id - this._clientId = clientID; + this._clientId = clientId; this._id = uuid(); this._status = 'unknown'; this._statusChanged = new Signal(this); - this.jmpConnection = jmpConnection; - } - public async connect(connectInfo: IJMPConnectionInfo) { - await this.jmpConnection.connect(connectInfo); + // Subscribe to messages coming in from our JMP channel + this.jmpConnection = jmpConnection; this.jmpConnection.subscribe((message) => { this.msgIn(message); }); diff --git a/src/client/datascience/raw-kernel/rawNotebookProviderWrapper.ts b/src/client/datascience/raw-kernel/rawNotebookProviderWrapper.ts index 7ff7bbba5e6b..2cd076ecb814 100644 --- a/src/client/datascience/raw-kernel/rawNotebookProviderWrapper.ts +++ b/src/client/datascience/raw-kernel/rawNotebookProviderWrapper.ts @@ -13,6 +13,7 @@ import { IAsyncDisposableRegistry, IConfigurationService, IDisposableRegistry, R import { IServiceContainer } from '../../ioc/types'; import { IRoleBasedObject, RoleBasedFactory } from '../jupyter/liveshare/roleBasedFactory'; import { ILiveShareHasRole } from '../jupyter/liveshare/types'; +import { IKernelLauncher } from '../kernel-launcher/types'; import { INotebook, IRawConnection, IRawNotebookProvider } from '../types'; import { GuestRawNotebookProvider } from './liveshare/guestRawNotebookProvider'; import { HostRawNotebookProvider } from './liveshare/hostRawNotebookProvider'; @@ -29,7 +30,8 @@ type RawNotebookProviderClassType = { workspaceService: IWorkspaceService, appShell: IApplicationShell, fs: IFileSystem, - serviceContainer: IServiceContainer + serviceContainer: IServiceContainer, + kernelLauncher: IKernelLauncher ): IRawNotebookProviderInterface; }; // tslint:enable:callable-types @@ -48,7 +50,8 @@ export class RawNotebookProviderWrapper implements IRawNotebookProvider, ILiveSh @inject(IWorkspaceService) workspaceService: IWorkspaceService, @inject(IApplicationShell) appShell: IApplicationShell, @inject(IFileSystem) fs: IFileSystem, - @inject(IServiceContainer) serviceContainer: IServiceContainer + @inject(IServiceContainer) serviceContainer: IServiceContainer, + @inject(IKernelLauncher) kernelLauncher: IKernelLauncher ) { // The server factory will create the appropriate HostRawNotebookProvider or GuestRawNotebookProvider based on // the liveshare state. @@ -63,7 +66,8 @@ export class RawNotebookProviderWrapper implements IRawNotebookProvider, ILiveSh workspaceService, appShell, fs, - serviceContainer + serviceContainer, + kernelLauncher ); } diff --git a/src/client/datascience/raw-kernel/rawSession.ts b/src/client/datascience/raw-kernel/rawSession.ts index dcfff0f1f09d..e79cb6b7abbc 100644 --- a/src/client/datascience/raw-kernel/rawSession.ts +++ b/src/client/datascience/raw-kernel/rawSession.ts @@ -3,7 +3,7 @@ import { Kernel, KernelMessage, ServerConnection, Session } from '@jupyterlab/services'; import { ISignal, Signal } from '@phosphor/signaling'; import * as uuid from 'uuid/v4'; -import { IJMPConnection, IJMPConnectionInfo } from '../types'; +import { IJMPConnection } from '../types'; import { RawKernel } from './rawKernel'; /* @@ -22,8 +22,6 @@ export class RawSession implements Session.ISession { private _kernel: RawKernel; private _statusChanged = new Signal(this); - // RAWKERNEL: Still just pass connection for now, we'll have to - // inject this further up the chain constructor(connection: IJMPConnection) { // Unique ID for this session instance this._id = uuid(); @@ -31,14 +29,8 @@ export class RawSession implements Session.ISession { // ID for our client JMP connection this._clientID = uuid(); - // Connect our kernel + // Connect our kernel and hook up status changes this._kernel = new RawKernel(connection, this._clientID); - } - - public async connect(connectionInfo: IJMPConnectionInfo) { - await this._kernel.connect(connectionInfo); - - // Connect for status changes this._kernel.statusChanged.connect(this.onKernelStatus, this); } diff --git a/src/client/datascience/utils.ts b/src/client/datascience/utils.ts new file mode 100644 index 000000000000..ff272da36163 --- /dev/null +++ b/src/client/datascience/utils.ts @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +'use strict'; + +import * as path from 'path'; + +import { IWorkspaceService } from '../common/application/types'; +import { IFileSystem } from '../common/platform/types'; +import { IConfigurationService } from '../common/types'; + +export async function calculateWorkingDirectory( + configService: IConfigurationService, + workspace: IWorkspaceService, + fileSystem: IFileSystem +): Promise { + let workingDir: string | undefined; + // For a local launch calculate the working directory that we should switch into + const settings = configService.getSettings(undefined); + const fileRoot = settings.datascience.notebookFileRoot; + + // If we don't have a workspace open the notebookFileRoot seems to often have a random location in it (we use ${workspaceRoot} as default) + // so only do this setting if we actually have a valid workspace open + if (fileRoot && workspace.hasWorkspaceFolders) { + const workspaceFolderPath = workspace.workspaceFolders![0].uri.fsPath; + if (path.isAbsolute(fileRoot)) { + if (await fileSystem.directoryExists(fileRoot)) { + // User setting is absolute and exists, use it + workingDir = fileRoot; + } else { + // User setting is absolute and doesn't exist, use workspace + workingDir = workspaceFolderPath; + } + } else if (!fileRoot.includes('${')) { + // fileRoot is a relative path, combine it with the workspace folder + const combinedPath = path.join(workspaceFolderPath, fileRoot); + if (await fileSystem.directoryExists(combinedPath)) { + // combined path exists, use it + workingDir = combinedPath; + } else { + // Combined path doesn't exist, use workspace + workingDir = workspaceFolderPath; + } + } else { + // fileRoot is a variable that hasn't been expanded + workingDir = fileRoot; + } + } + return workingDir; +} diff --git a/src/test/datascience/raw-kernel/rawJupyterSession.unit.test.ts b/src/test/datascience/raw-kernel/rawJupyterSession.unit.test.ts index 64a91155e72d..2f93ec6cef0a 100644 --- a/src/test/datascience/raw-kernel/rawJupyterSession.unit.test.ts +++ b/src/test/datascience/raw-kernel/rawJupyterSession.unit.test.ts @@ -1,40 +1,55 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -import { assert } from 'chai'; +import { assert, expect } from 'chai'; import * as sinon from 'sinon'; -import { mock } from 'ts-mockito'; +import { anything, instance, mock, when } from 'ts-mockito'; +import * as typemoq from 'typemoq'; +import { EventEmitter } from 'vscode'; +import { IKernelLauncher, IKernelProcess } from '../../../client/datascience/kernel-launcher/types'; import { RawJupyterSession } from '../../../client/datascience/raw-kernel/rawJupyterSession'; -import { IJMPConnection, IJMPConnectionInfo } from '../../../client/datascience/types'; +import { IJMPConnection } from '../../../client/datascience/types'; +import { IServiceContainer } from '../../../client/ioc/types'; + +// tslint:disable:no-any +function createTypeMoq(tag: string): typemoq.IMock { + // Use typemoqs for those things that are resolved as promises. mockito doesn't allow nesting of mocks. ES6 Proxy class + // is the problem. We still need to make it thenable though. See this issue: https://github.com/florinn/typemoq/issues/67 + const result = typemoq.Mock.ofType(); + (result as any).tag = tag; + result.setup((x: any) => x.then).returns(() => undefined); + return result; +} // Note: The jupyterSession.unit.test.ts tests cover much of the base class functionality // and lower level is handled by RawFuture, RawKernel, and RawSession // tslint:disable: max-func-body-length suite('Data Science - RawJupyterSession', () => { let rawJupyterSession: RawJupyterSession; - let jmpConnection: IJMPConnection; - let connectInfo: IJMPConnectionInfo; + let serviceContainer: IServiceContainer; + let kernelLauncher: IKernelLauncher; + let jmpConnection: typemoq.IMock; + let kernelProcess: typemoq.IMock; + let processExitEvent: EventEmitter; setup(() => { - jmpConnection = mock(); - connectInfo = { - version: 0, - transport: 'tcp', - ip: '127.0.0.1', - shell_port: 55196, - iopub_port: 55197, - stdin_port: 55198, - hb_port: 55200, - control_port: 55199, - signature_scheme: 'hmac-sha256', - key: 'adaf9032-487d222a85026db284c3d5e7' - }; - rawJupyterSession = new RawJupyterSession(jmpConnection); - }); + serviceContainer = mock(); + kernelLauncher = mock(); - test('RawJupyterSession - connect', async () => { - await rawJupyterSession.connect(connectInfo); + // Fake out our jmp connection + jmpConnection = createTypeMoq('jmp connection'); + jmpConnection.setup(jmp => jmp.connect(typemoq.It.isAny())).returns(() => Promise.resolve()); + when(serviceContainer.get(IJMPConnection)).thenReturn(jmpConnection.object); + + // Set up a fake kernel process for the launcher to return + processExitEvent = new EventEmitter(); + kernelProcess = createTypeMoq('kernel process'); + kernelProcess.setup(kp => kp.kernelSpec).returns(() => 'testspec' as any); + kernelProcess.setup(kp => kp.connection).returns(() => 'testconnection' as any); + kernelProcess.setup(kp => kp.ready).returns(() => Promise.resolve()); + kernelProcess.setup(kp => kp.exited).returns(() => processExitEvent.event); + when(kernelLauncher.launch(anything(), anything())).thenResolve(kernelProcess.object); - assert.isTrue(rawJupyterSession.isConnected); + rawJupyterSession = new RawJupyterSession(instance(kernelLauncher), instance(serviceContainer)); }); test('RawJupyterSession - shutdown on dispose', async () => { @@ -43,4 +58,23 @@ suite('Data Science - RawJupyterSession', () => { await rawJupyterSession.dispose(); assert.isTrue(shutdown.calledOnce); }); + + test('RawJupyterSession - connect', async () => { + await rawJupyterSession.connect({} as any, 60_000); + expect(rawJupyterSession.isConnected).to.equal(true, 'RawJupyterSession not connected'); + }); + + test('RawJupyterSession - Kill process', async () => { + const shutdown = sinon.stub(rawJupyterSession, 'shutdown'); + shutdown.resolves(); + + const kernelSpec = await rawJupyterSession.connect({} as any, 60_000); + expect(rawJupyterSession.isConnected).to.equal(true, 'RawJupyterSession not connected'); + expect(kernelSpec).to.equal('testspec'); + + // Kill the process, we should shutdown + processExitEvent.fire(0); + + assert.isTrue(shutdown.calledOnce); + }); }); diff --git a/src/test/datascience/raw-kernel/rawKernel.unit.test.ts b/src/test/datascience/raw-kernel/rawKernel.unit.test.ts index 1c447041210f..bd7a571215d4 100644 --- a/src/test/datascience/raw-kernel/rawKernel.unit.test.ts +++ b/src/test/datascience/raw-kernel/rawKernel.unit.test.ts @@ -3,17 +3,16 @@ import { Kernel, KernelMessage } from '@jupyterlab/services'; import { Slot } from '@phosphor/signaling'; import { assert, expect } from 'chai'; -import { anything, deepEqual, instance, mock, verify, when } from 'ts-mockito'; +import { anything, instance, mock, verify, when } from 'ts-mockito'; import * as uuid from 'uuid/v4'; import { RawKernel } from '../../../client/datascience/raw-kernel/rawKernel'; -import { IJMPConnection, IJMPConnectionInfo } from '../../../client/datascience/types'; +import { IJMPConnection } from '../../../client/datascience/types'; import { MockJMPConnection } from './mockJMP'; // tslint:disable: max-func-body-length suite('Data Science - RawKernel', () => { let rawKernel: RawKernel; let jmpConnection: IJMPConnection; - let connectInfo: IJMPConnectionInfo; suite('RawKernel basic mock jmp', () => { setup(() => { @@ -21,24 +20,9 @@ suite('Data Science - RawKernel', () => { when(jmpConnection.connect(anything())).thenResolve(); when(jmpConnection.subscribe(anything())).thenReturn(); rawKernel = new RawKernel(instance(jmpConnection), uuid()); - - connectInfo = { - version: 0, - transport: 'tcp', - ip: '127.0.0.1', - shell_port: 55196, - iopub_port: 55197, - stdin_port: 55198, - hb_port: 55200, - control_port: 55199, - signature_scheme: 'hmac-sha256', - key: 'adaf9032-487d222a85026db284c3d5e7' - }; }); test('RawKernel connect should connect and subscribe to JMP', async () => { - await rawKernel.connect(connectInfo); - verify(jmpConnection.connect(deepEqual(connectInfo))).once(); verify(jmpConnection.subscribe(anything())).once(); // Verify that we have a client id an a kernel id expect(rawKernel.id).to.not.equal(rawKernel.clientId); @@ -47,8 +31,6 @@ suite('Data Science - RawKernel', () => { test('RawKernel dispose should dispose the jmp', async () => { when(jmpConnection.dispose()).thenReturn(); - await rawKernel.connect(connectInfo); - // Dispose our kernel rawKernel.dispose(); @@ -59,8 +41,6 @@ suite('Data Science - RawKernel', () => { test('RawKernel requestExecute should pass a valid execute message to JMP', async () => { when(jmpConnection.sendMessage(anything())).thenReturn(); - await rawKernel.connect(connectInfo); - const code = 'print("hello world")'; const executeContent: KernelMessage.IExecuteRequestMsg['content'] = { code @@ -81,8 +61,6 @@ suite('Data Science - RawKernel', () => { when(jmpConnection.sendMessage(anything())).thenReturn(); when(jmpConnection.dispose()).thenReturn(); - await rawKernel.connect(connectInfo); - const code = 'print("hello world")'; const executeContent: KernelMessage.IExecuteRequestMsg['content'] = { code @@ -110,8 +88,6 @@ suite('Data Science - RawKernel', () => { }); test('RawKernel executeRequest messages', async () => { - await rawKernel.connect(connectInfo); - // Check our status at the start expect(rawKernel.status).to.equal('unknown'); @@ -202,8 +178,6 @@ suite('Data Science - RawKernel', () => { }); test('RawKernel requestInspect messages', async () => { - await rawKernel.connect(connectInfo); - // Check our status at the start expect(rawKernel.status).to.equal('unknown'); @@ -240,8 +214,6 @@ suite('Data Science - RawKernel', () => { }); test('RawKernel requestComplete messages', async () => { - await rawKernel.connect(connectInfo); - // Check our status at the start expect(rawKernel.status).to.equal('unknown'); @@ -277,8 +249,6 @@ suite('Data Science - RawKernel', () => { }); test('RawKernel sendInput messages', async () => { - await rawKernel.connect(connectInfo); - // Check our status at the start expect(rawKernel.status).to.equal('unknown'); @@ -303,7 +273,6 @@ suite('Data Science - RawKernel', () => { // update_display_data message test('rawKernel displayid check', async () => { const displayId = '1'; - await rawKernel.connect(connectInfo); // Check our status at the start expect(rawKernel.status).to.equal('unknown'); diff --git a/src/test/datascience/raw-kernel/rawSession.unit.test.ts b/src/test/datascience/raw-kernel/rawSession.unit.test.ts index ae0893f98e0f..5c1088e51532 100644 --- a/src/test/datascience/raw-kernel/rawSession.unit.test.ts +++ b/src/test/datascience/raw-kernel/rawSession.unit.test.ts @@ -3,16 +3,15 @@ import { Kernel, KernelMessage } from '@jupyterlab/services'; import { Slot } from '@phosphor/signaling'; import { expect } from 'chai'; -import { anything, deepEqual, instance, mock, verify, when } from 'ts-mockito'; +import { anything, instance, mock, verify, when } from 'ts-mockito'; import { RawSession } from '../../../client/datascience/raw-kernel/rawSession'; -import { IJMPConnection, IJMPConnectionInfo } from '../../../client/datascience/types'; +import { IJMPConnection } from '../../../client/datascience/types'; import { MockJMPConnection } from './mockJMP'; import { buildStatusMessage } from './rawKernel.unit.test'; // tslint:disable: max-func-body-length suite('Data Science - RawSession', () => { let rawSession: RawSession; - let connectInfo: IJMPConnectionInfo; suite('RawSession - basic JMP', () => { let jmpConnection: IJMPConnection; @@ -21,19 +20,6 @@ suite('Data Science - RawSession', () => { when(jmpConnection.connect(anything())).thenResolve(); when(jmpConnection.subscribe(anything())).thenReturn(); rawSession = new RawSession(instance(jmpConnection)); - - connectInfo = { - version: 0, - transport: 'tcp', - ip: '127.0.0.1', - shell_port: 55196, - iopub_port: 55197, - stdin_port: 55198, - hb_port: 55200, - control_port: 55199, - signature_scheme: 'hmac-sha256', - key: 'adaf9032-487d222a85026db284c3d5e7' - }; }); test('RawSession construct', async () => { @@ -45,10 +31,7 @@ suite('Data Science - RawSession', () => { }); test('RawSession connect', async () => { - await rawSession.connect(connectInfo); - // Did we hook up our connection - verify(jmpConnection.connect(deepEqual(connectInfo))).once(); verify(jmpConnection.subscribe(anything())).once(); // The ID of the session is not the same as the kernel client id expect(rawSession.kernel.clientId).to.not.equal(rawSession.id); @@ -70,24 +53,9 @@ suite('Data Science - RawSession', () => { setup(() => { mockJmpConnection = new MockJMPConnection(); rawSession = new RawSession(mockJmpConnection); - - connectInfo = { - version: 0, - transport: 'tcp', - ip: '127.0.0.1', - shell_port: 55196, - iopub_port: 55197, - stdin_port: 55198, - hb_port: 55200, - control_port: 55199, - signature_scheme: 'hmac-sha256', - key: 'adaf9032-487d222a85026db284c3d5e7' - }; }); test('RawSession status updates', async () => { - await rawSession.connect(connectInfo); - const statusChanges = ['busy', 'idle']; let statusHit = 0; const statusHandler: Slot = (_sender: RawSession, args: Kernel.Status) => { From 96d16456bad5a1007e1416ebb6c4a663e2ec280d Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Wed, 15 Apr 2020 10:24:22 -0700 Subject: [PATCH 016/725] prettier fixup --- .../datascience/kernel-launcher/kernelLauncher.ts | 4 ++-- .../raw-kernel/liveshare/hostRawNotebookProvider.ts | 2 +- src/client/datascience/raw-kernel/rawJupyterSession.ts | 4 ++-- .../datascience/raw-kernel/rawNotebookProvider.ts | 4 ++-- .../raw-kernel/rawJupyterSession.unit.test.ts | 10 +++++----- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/client/datascience/kernel-launcher/kernelLauncher.ts b/src/client/datascience/kernel-launcher/kernelLauncher.ts index ce0e9426a2af..9ceb8c61681b 100644 --- a/src/client/datascience/kernel-launcher/kernelLauncher.ts +++ b/src/client/datascience/kernel-launcher/kernelLauncher.ts @@ -79,7 +79,7 @@ class KernelProcess implements IKernelProcess { const exeObs = executionService.execObservable(args, {}); if (exeObs.proc) { - exeObs.proc!.on('exit', exitCode => { + exeObs.proc!.on('exit', (exitCode) => { traceInfo('KernelProcess Exit', `Exit - ${exitCode}`); if (!this.readyPromise.completed) { this.readyPromise.reject(new Error(localize.DataScience.rawKernelProcessExitBeforeConnect())); @@ -90,7 +90,7 @@ class KernelProcess implements IKernelProcess { traceInfo('KernelProcess failed to launch'); this.readyPromise.reject(new Error(localize.DataScience.rawKernelProcessNotStarted())); } - exeObs.out.subscribe(output => { + exeObs.out.subscribe((output) => { if (output.source === 'stderr') { traceWarning(`StdErr from Kernel Process ${output.out}`); } else { diff --git a/src/client/datascience/raw-kernel/liveshare/hostRawNotebookProvider.ts b/src/client/datascience/raw-kernel/liveshare/hostRawNotebookProvider.ts index f2e7f2a9d26b..af588efbc123 100644 --- a/src/client/datascience/raw-kernel/liveshare/hostRawNotebookProvider.ts +++ b/src/client/datascience/raw-kernel/liveshare/hostRawNotebookProvider.ts @@ -124,7 +124,7 @@ export class HostRawNotebookProvider } } catch (ex) { // Make sure we shut down our session in case we started a process - rawSession.dispose().catch(error => { + rawSession.dispose().catch((error) => { traceError(`Failed to dispose of raw session on launch error: ${error} `); }); // If there's an error, then reject the promise that is returned. diff --git a/src/client/datascience/raw-kernel/rawJupyterSession.ts b/src/client/datascience/raw-kernel/rawJupyterSession.ts index bfaba6f5bff7..25afeee37597 100644 --- a/src/client/datascience/raw-kernel/rawJupyterSession.ts +++ b/src/client/datascience/raw-kernel/rawJupyterSession.ts @@ -122,9 +122,9 @@ export class RawJupyterSession extends BaseJupyterSession { } // Watch to see if our process exits - this.processExitHandler = process.exited(exitCode => { + this.processExitHandler = process.exited((exitCode) => { traceError(`Raw kernel process exited code: ${exitCode}`); - this.shutdown().catch(reason => { + this.shutdown().catch((reason) => { traceError(`Error shutting down raw jupyter session: ${reason}`); }); // Next code the user executes will show a session disposed message diff --git a/src/client/datascience/raw-kernel/rawNotebookProvider.ts b/src/client/datascience/raw-kernel/rawNotebookProvider.ts index 8cfee8c5b0c0..1946736d2f34 100644 --- a/src/client/datascience/raw-kernel/rawNotebookProvider.ts +++ b/src/client/datascience/raw-kernel/rawNotebookProvider.ts @@ -61,7 +61,7 @@ export class RawNotebookProviderBase implements IRawNotebookProvider { public async dispose(): Promise { traceInfo(`Shutting down notebooks for ${this.id}`); const notebooks = await Promise.all([...this.notebooks.values()]); - await Promise.all(notebooks.map(n => n?.dispose())); + await Promise.all(notebooks.map((n) => n?.dispose())); } // This may be a bit of a noop in the raw case @@ -81,7 +81,7 @@ export class RawNotebookProviderBase implements IRawNotebookProvider { }; notebook - .then(nb => { + .then((nb) => { const oldDispose = nb.dispose; nb.dispose = () => { this.notebooks.delete(identity.toString()); diff --git a/src/test/datascience/raw-kernel/rawJupyterSession.unit.test.ts b/src/test/datascience/raw-kernel/rawJupyterSession.unit.test.ts index 2f93ec6cef0a..5e11bd47d054 100644 --- a/src/test/datascience/raw-kernel/rawJupyterSession.unit.test.ts +++ b/src/test/datascience/raw-kernel/rawJupyterSession.unit.test.ts @@ -37,16 +37,16 @@ suite('Data Science - RawJupyterSession', () => { // Fake out our jmp connection jmpConnection = createTypeMoq('jmp connection'); - jmpConnection.setup(jmp => jmp.connect(typemoq.It.isAny())).returns(() => Promise.resolve()); + jmpConnection.setup((jmp) => jmp.connect(typemoq.It.isAny())).returns(() => Promise.resolve()); when(serviceContainer.get(IJMPConnection)).thenReturn(jmpConnection.object); // Set up a fake kernel process for the launcher to return processExitEvent = new EventEmitter(); kernelProcess = createTypeMoq('kernel process'); - kernelProcess.setup(kp => kp.kernelSpec).returns(() => 'testspec' as any); - kernelProcess.setup(kp => kp.connection).returns(() => 'testconnection' as any); - kernelProcess.setup(kp => kp.ready).returns(() => Promise.resolve()); - kernelProcess.setup(kp => kp.exited).returns(() => processExitEvent.event); + kernelProcess.setup((kp) => kp.kernelSpec).returns(() => 'testspec' as any); + kernelProcess.setup((kp) => kp.connection).returns(() => 'testconnection' as any); + kernelProcess.setup((kp) => kp.ready).returns(() => Promise.resolve()); + kernelProcess.setup((kp) => kp.exited).returns(() => processExitEvent.event); when(kernelLauncher.launch(anything(), anything())).thenResolve(kernelProcess.object); rawJupyterSession = new RawJupyterSession(instance(kernelLauncher), instance(serviceContainer)); From deaecfb0343ab8dc28884b1818b0ec48b6d0b7d4 Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Wed, 15 Apr 2020 10:36:43 -0700 Subject: [PATCH 017/725] update ui dependencies --- package.datascience-ui.dependencies.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.datascience-ui.dependencies.json b/package.datascience-ui.dependencies.json index c80a6d3e5a04..f964e6c24df9 100644 --- a/package.datascience-ui.dependencies.json +++ b/package.datascience-ui.dependencies.json @@ -220,6 +220,7 @@ "ripemd160", "roughjs-es5", "rxjs", + "rxjs-compat", "safe-buffer", "scheduler", "semiotic", From ee23a4104b0678aaa1ecb5d342ecaa7106abeb7d Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Wed, 15 Apr 2020 11:14:07 -0700 Subject: [PATCH 018/725] turn off experiments in ioc --- src/test/datascience/dataScienceIocContainer.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/datascience/dataScienceIocContainer.ts b/src/test/datascience/dataScienceIocContainer.ts index 769179446b1d..36140b1888af 100644 --- a/src/test/datascience/dataScienceIocContainer.ts +++ b/src/test/datascience/dataScienceIocContainer.ts @@ -827,9 +827,9 @@ export class DataScienceIocContainer extends UnitTestIocContainer { instance(packageService) ); - // Enable experiments. + // Turn off experiments. const experimentManager = mock(ExperimentsManager); - when(experimentManager.inExperiment(anything())).thenReturn(true); + when(experimentManager.inExperiment(anything())).thenReturn(false); when(experimentManager.activate()).thenResolve(); this.serviceManager.addSingletonInstance(IExperimentsManager, instance(experimentManager)); From 7846e308fc79ed9c1a287720076389ad2b6d6c65 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Wed, 15 Apr 2020 11:45:05 -0700 Subject: [PATCH 019/725] Allow interrupting the kernel more than once (#11184) For #10587, #10356 --- news/2 Fixes/10356.md | 1 + news/2 Fixes/10587.md | 1 + .../interactive-common/interactiveBase.ts | 48 ++++++++++++------- .../redux/reducers/kernel.ts | 12 +---- 4 files changed, 35 insertions(+), 27 deletions(-) create mode 100644 news/2 Fixes/10356.md create mode 100644 news/2 Fixes/10587.md diff --git a/news/2 Fixes/10356.md b/news/2 Fixes/10356.md new file mode 100644 index 000000000000..4c557190d24c --- /dev/null +++ b/news/2 Fixes/10356.md @@ -0,0 +1 @@ +Cancelling the prompt to restart the kernel should not leave the toolbar buttons disabled. diff --git a/news/2 Fixes/10587.md b/news/2 Fixes/10587.md new file mode 100644 index 000000000000..2dde39578bf3 --- /dev/null +++ b/news/2 Fixes/10587.md @@ -0,0 +1 @@ +Allow interrupting the kernel more than once. diff --git a/src/client/datascience/interactive-common/interactiveBase.ts b/src/client/datascience/interactive-common/interactiveBase.ts index 20798c17f491..81ae0d620c5b 100644 --- a/src/client/datascience/interactive-common/interactiveBase.ts +++ b/src/client/datascience/interactive-common/interactiveBase.ts @@ -381,22 +381,30 @@ export abstract class InteractiveBase extends WebViewHost { if (this._notebook && !this.restartingKernel) { + this.restartingKernel = true; + this.startProgress(); + const status = this.statusProvider.set( localize.DataScience.interruptKernelStatus(), true, @@ -412,10 +423,10 @@ export abstract class InteractiveBase extends WebViewHost Date: Wed, 15 Apr 2020 11:46:41 -0700 Subject: [PATCH 020/725] Retry ipywidget tests (#11189) A number of the IPyWidget tests are failing on the flaky pipeline. Ensured tests are re-tried. --- src/test/datascience/uiTests/ipywidget.ui.functional.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/datascience/uiTests/ipywidget.ui.functional.test.ts b/src/test/datascience/uiTests/ipywidget.ui.functional.test.ts index 3e006dfc1c53..3d67d3990f78 100644 --- a/src/test/datascience/uiTests/ipywidget.ui.functional.test.ts +++ b/src/test/datascience/uiTests/ipywidget.ui.functional.test.ts @@ -37,6 +37,7 @@ use(chaiAsPromised); suiteSetup(function () { // These are UI tests, hence nothing to do with platforms. this.timeout(30_000); // UI Tests, need time to start jupyter. + this.retries(3); // UI tests can be flaky. if (!process.env.VSCODE_PYTHON_ROLLING) { // Skip all tests unless using real jupyter this.skip(); From 02ff34525383994a08e514f579f2625d1a4942e8 Mon Sep 17 00:00:00 2001 From: David Kutugata Date: Wed, 15 Apr 2020 11:51:34 -0700 Subject: [PATCH 021/725] When pressing ctrl+enter on a markdown cell, unfocus it so it renders (#11162) * When pressing ctrl+enter on a markdown cell, unfocus it so it renders * add news file * changed a comment * reuse the function to escape the cell --- news/2 Fixes/10006.md | 1 + src/datascience-ui/native-editor/nativeCell.tsx | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 news/2 Fixes/10006.md diff --git a/news/2 Fixes/10006.md b/news/2 Fixes/10006.md new file mode 100644 index 000000000000..bbd18ba9b550 --- /dev/null +++ b/news/2 Fixes/10006.md @@ -0,0 +1 @@ +Fix ctrl+enter on markdown cells. Now they render. diff --git a/src/datascience-ui/native-editor/nativeCell.tsx b/src/datascience-ui/native-editor/nativeCell.tsx index 07ff608fd009..cd4ca37a600c 100644 --- a/src/datascience-ui/native-editor/nativeCell.tsx +++ b/src/datascience-ui/native-editor/nativeCell.tsx @@ -449,6 +449,11 @@ export class NativeCell extends React.Component { e.stopPropagation(); e.preventDefault(); + // Escape the current cell if it is markdown to make it render + if (this.isMarkdownCell()) { + this.escapeCell(e); + } + // Submit this cell this.submitCell('none'); this.props.sendCommand(NativeCommandType.Run, 'keyboard'); From 4b3dff08e849f7fd52221eb37ce061febd3bdb6f Mon Sep 17 00:00:00 2001 From: Kartik Raj Date: Thu, 16 Apr 2020 00:49:38 +0530 Subject: [PATCH 022/725] Fix bug with autoselection (#11185) --- src/client/interpreter/autoSelection/index.ts | 9 --------- .../autoSelection/index.unit.test.ts | 20 ------------------- 2 files changed, 29 deletions(-) diff --git a/src/client/interpreter/autoSelection/index.ts b/src/client/interpreter/autoSelection/index.ts index 1d59b1bb8387..0c275096e0c7 100644 --- a/src/client/interpreter/autoSelection/index.ts +++ b/src/client/interpreter/autoSelection/index.ts @@ -135,15 +135,6 @@ export class InterpreterAutoSelectionService implements IInterpreterAutoSelectio return this.globallyPreferredInterpreter.value; } public async setWorkspaceInterpreter(resource: Uri, interpreter: PythonInterpreter | undefined) { - // We can only update the stored interpreter once we have done the necessary - // work of auto selecting the interpreters. - if ( - !this.autoSelectedWorkspacePromises.has(this.getWorkspacePathKey(resource)) || - !this.autoSelectedWorkspacePromises.get(this.getWorkspacePathKey(resource))!.completed - ) { - return; - } - await this.storeAutoSelectedInterpreter(resource, interpreter); } public async setGlobalInterpreter(interpreter: PythonInterpreter) { diff --git a/src/test/interpreters/autoSelection/index.unit.test.ts b/src/test/interpreters/autoSelection/index.unit.test.ts index 369fb71c5d93..cbe397648a1e 100644 --- a/src/test/interpreters/autoSelection/index.unit.test.ts +++ b/src/test/interpreters/autoSelection/index.unit.test.ts @@ -319,26 +319,6 @@ suite('Interpreters - Auto Selection', () => { expect(selectedInterpreter).to.deep.equal(interpreterInfo); expect(eventFired).to.deep.equal(false, 'event fired'); }); - test('Storing workspace interpreter info in state store should fail', async () => { - const pythonPath = 'Hello World'; - const interpreterInfo = { path: pythonPath } as any; - const resource = Uri.parse('one'); - when( - stateFactory.createGlobalPersistentState( - preferredGlobalInterpreter, - undefined - ) - ).thenReturn(instance(state)); - when(workspaceService.getWorkspaceFolder(resource)).thenReturn({ name: '', index: 0, uri: resource }); - when(workspaceService.getWorkspaceFolderIdentifier(anything(), anything())).thenReturn(''); - - await autoSelectionService.initializeStore(undefined); - await autoSelectionService.setWorkspaceInterpreter(resource, interpreterInfo); - const selectedInterpreter = autoSelectionService.getAutoSelectedInterpreter(resource); - - verify(state.updateValue(interpreterInfo)).never(); - expect(selectedInterpreter ? selectedInterpreter : undefined).to.deep.equal(undefined, 'not undefined'); - }); test('Store workspace interpreter info in state store', async () => { const pythonPath = 'Hello World'; const interpreterInfo = { path: pythonPath } as any; From 51cd8d20a42f2098cb89a7425e241755aee76f3c Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Wed, 15 Apr 2020 12:39:02 -0700 Subject: [PATCH 023/725] Ensure save works after saving untitled notebooks (#11190) * Ensure save works after saving untitled notebooks * Remove old document --- .../datascience/interactive-ipynb/nativeEditorProviderOld.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/client/datascience/interactive-ipynb/nativeEditorProviderOld.ts b/src/client/datascience/interactive-ipynb/nativeEditorProviderOld.ts index 1ed7109a3f10..fcdcea4afa2a 100644 --- a/src/client/datascience/interactive-ipynb/nativeEditorProviderOld.ts +++ b/src/client/datascience/interactive-ipynb/nativeEditorProviderOld.ts @@ -66,6 +66,8 @@ export class NativeEditorProviderOld extends NativeEditorProvider { const customDocument = this.customDocuments.get(resource.fsPath); if (customDocument) { await this.saveAs(customDocument, targetResource); + this.customDocuments.delete(resource.fsPath); + this.customDocuments.set(targetResource.fsPath, { ...customDocument, uri: targetResource }); } } ) From 1b0894fd903447d553781ba58fc550bdb718e8a6 Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Wed, 15 Apr 2020 12:58:55 -0700 Subject: [PATCH 024/725] Fix failing unit tests on windows (#11191) --- .../synchronousTerminalService.unit.test.ts | 18 ++---------------- ...alEnvironmentActivationService.unit.test.ts | 6 +----- 2 files changed, 3 insertions(+), 21 deletions(-) diff --git a/src/test/common/terminals/synchronousTerminalService.unit.test.ts b/src/test/common/terminals/synchronousTerminalService.unit.test.ts index 4d0b6be2af0d..082eeb1953ca 100644 --- a/src/test/common/terminals/synchronousTerminalService.unit.test.ts +++ b/src/test/common/terminals/synchronousTerminalService.unit.test.ts @@ -107,14 +107,7 @@ suite('Terminal Service (synchronous)', () => { verify( terminalService.sendCommand( 'python', - deepEqual([ - isolated.fileToCommandArgument(), - shellExecFile.fileToCommandArgument(), - 'cmd'.fileToCommandArgument(), - '1', - '2', - tmpFile.filePath.fileToCommandArgument() - ]) + deepEqual([isolated, shellExecFile, 'cmd', '1', '2', tmpFile.filePath.fileToCommandArgument()]) ) ).once(); }).timeout(1_000); @@ -150,14 +143,7 @@ suite('Terminal Service (synchronous)', () => { verify( terminalService.sendCommand( 'python', - deepEqual([ - isolated.fileToCommandArgument(), - shellExecFile.fileToCommandArgument(), - 'cmd'.fileToCommandArgument(), - '1', - '2', - tmpFile.filePath.fileToCommandArgument() - ]) + deepEqual([isolated, shellExecFile, 'cmd', '1', '2', tmpFile.filePath.fileToCommandArgument()]) ) ).once(); }).timeout(2_000); diff --git a/src/test/interpreters/activation/terminalEnvironmentActivationService.unit.test.ts b/src/test/interpreters/activation/terminalEnvironmentActivationService.unit.test.ts index 5789716813c6..96400cf909a2 100644 --- a/src/test/interpreters/activation/terminalEnvironmentActivationService.unit.test.ts +++ b/src/test/interpreters/activation/terminalEnvironmentActivationService.unit.test.ts @@ -104,11 +104,7 @@ suite('Interpreters Activation - Python Environment Variables (using terminals)' verify( terminal.sendCommand( cmd, - deepEqual([ - isolated.fileToCommandArgument(), - pyFile.fileToCommandArgument(), - jsonFile.fileToCommandArgument() - ]), + deepEqual([isolated, pyFile, jsonFile.fileToCommandArgument()]), anything(), false ) From a9a78eae5403ac2e99fab433aed5ab0bb8f479da Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Wed, 15 Apr 2020 13:27:42 -0700 Subject: [PATCH 025/725] experiments on or off in ioc --- src/test/datascience/dataScienceIocContainer.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/test/datascience/dataScienceIocContainer.ts b/src/test/datascience/dataScienceIocContainer.ts index 36140b1888af..b5d7fb602a2c 100644 --- a/src/test/datascience/dataScienceIocContainer.ts +++ b/src/test/datascience/dataScienceIocContainer.ts @@ -90,6 +90,7 @@ import { EXTENSION_ROOT_DIR, UseCustomEditorApi } from '../../client/common/cons import { CryptoUtils } from '../../client/common/crypto'; import { DotNetCompatibilityService } from '../../client/common/dotnet/compatibilityService'; import { IDotNetCompatibilityService } from '../../client/common/dotnet/types'; +import { LocalZMQKernel } from '../../client/common/experimentGroups'; import { ExperimentsManager } from '../../client/common/experiments'; import { InstallationChannelManager } from '../../client/common/installer/channelManager'; import { ProductInstaller } from '../../client/common/installer/productInstaller'; @@ -829,7 +830,12 @@ export class DataScienceIocContainer extends UnitTestIocContainer { // Turn off experiments. const experimentManager = mock(ExperimentsManager); - when(experimentManager.inExperiment(anything())).thenReturn(false); + when(experimentManager.inExperiment(anything())).thenCall((exp) => { + if (exp === LocalZMQKernel.experiment) { + return false; + } + return true; + }); when(experimentManager.activate()).thenResolve(); this.serviceManager.addSingletonInstance(IExperimentsManager, instance(experimentManager)); From 101f4855ffa62e78a174e01e5ef375805e80f978 Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Wed, 15 Apr 2020 13:34:17 -0700 Subject: [PATCH 026/725] Fix pdf viewer so that we just ship the standalone version. (#11192) --- build/webpack/common.js | 2 +- build/webpack/webpack.extension.config.js | 20 +++++++++---------- news/2 Fixes/11157.md | 1 + src/client/datascience/plotting/plotViewer.ts | 2 +- 4 files changed, 12 insertions(+), 13 deletions(-) create mode 100644 news/2 Fixes/11157.md diff --git a/build/webpack/common.js b/build/webpack/common.js index b06643759d39..53ee36a6077d 100644 --- a/build/webpack/common.js +++ b/build/webpack/common.js @@ -27,7 +27,7 @@ exports.nodeModulesToExternalize = [ 'node-stream-zip', 'xml2js', 'vsls/vscode', - 'pdfkit', + 'pdfkit/js/pdfkit.standalone', 'crypto-js', 'fontkit', 'linebreak', diff --git a/build/webpack/webpack.extension.config.js b/build/webpack/webpack.extension.config.js index f378beeb05be..63efdf641bd4 100644 --- a/build/webpack/webpack.extension.config.js +++ b/build/webpack/webpack.extension.config.js @@ -67,18 +67,16 @@ const config = { externals: ['vscode', 'commonjs', ...ppaPackageList, ...existingModulesInOutDir], plugins: [ ...common.getDefaultPlugins('extension'), - // Copy pdfkit bits after extension builds. webpack can't handle pdfkit. - new FileManagerPlugin({ - onEnd: [ - { - copy: [ - { source: './node_modules/fontkit/*.trie', destination: './out/client/node_modules' }, - { source: './node_modules/pdfkit/js/data/*.*', destination: './out/client/node_modules/data' }, - { source: './node_modules/pdfkit/js/pdfkit.js', destination: './out/client/node_modules/' } - ] - } - ] + // Copy pdfkit after extension builds. webpack can't handle pdfkit. + new removeFilesWebpackPlugin({ + after: { include: ['./out/client/node_modules/pdfkit/js/pdfkit.standalone.*'] } }), + new copyWebpackPlugin([ + { + from: './node_modules/pdfkit/js/pdfkit.standalone.js', + to: './node_modules/pdfkit/js/pdfkit.standalone.js' + } + ]), // ZMQ requires prebuilds to be in our node_modules directory. So recreate the ZMQ structure. // However we don't webpack to manage this, so it was part of the excluded modules. Delete it from there // so at runtime we pick up the original structure. diff --git a/news/2 Fixes/11157.md b/news/2 Fixes/11157.md new file mode 100644 index 000000000000..cffa03b0eeba --- /dev/null +++ b/news/2 Fixes/11157.md @@ -0,0 +1 @@ +Fix saving to PDF for viewed plots. \ No newline at end of file diff --git a/src/client/datascience/plotting/plotViewer.ts b/src/client/datascience/plotting/plotViewer.ts index 986079ff467e..5d9f64926c69 100644 --- a/src/client/datascience/plotting/plotViewer.ts +++ b/src/client/datascience/plotting/plotViewer.ts @@ -147,7 +147,7 @@ export class PlotViewer extends WebViewHost implements IPlot const SVGtoPDF = require('svg-to-pdfkit'); const deferred = createDeferred(); // tslint:disable-next-line: no-require-imports - const pdfkit = require('pdfkit') as typeof import('pdfkit'); + const pdfkit = require('pdfkit/js/pdfkit.standalone') as typeof import('pdfkit'); const doc = new pdfkit(); const ws = this.fileSystem.createWriteStream(file.fsPath); traceInfo(`Writing pdf to ${file.fsPath}`); From 42b332c402e96f849f7132e105b7e75ca9b6392e Mon Sep 17 00:00:00 2001 From: Ian Huff Date: Wed, 15 Apr 2020 13:39:19 -0700 Subject: [PATCH 027/725] don't keep checking on zmq support --- src/client/datascience/interactive-common/notebookProvider.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/datascience/interactive-common/notebookProvider.ts b/src/client/datascience/interactive-common/notebookProvider.ts index 5bc4cc7b21ea..e865ccb1f6cf 100644 --- a/src/client/datascience/interactive-common/notebookProvider.ts +++ b/src/client/datascience/interactive-common/notebookProvider.ts @@ -127,7 +127,7 @@ export class NotebookProvider implements INotebookProvider { // Check to see if this machine supports our local ZMQ launching private async zmqSupported(): Promise { - if (this._zmqSupported) { + if (this._zmqSupported !== undefined) { return this._zmqSupported; } From 5ae18618c811da1d53aec2eaa56e6c0c37e6a7ad Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Wed, 15 Apr 2020 15:10:39 -0700 Subject: [PATCH 028/725] Enable cell related commands when a Python file is already open (#11188) --- news/2 Fixes/10884.md | 1 + src/client/datascience/context/activeEditorContext.ts | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 news/2 Fixes/10884.md diff --git a/news/2 Fixes/10884.md b/news/2 Fixes/10884.md new file mode 100644 index 000000000000..a4582f1a0b9c --- /dev/null +++ b/news/2 Fixes/10884.md @@ -0,0 +1 @@ +Enable cell related commands when a Python file is already open. diff --git a/src/client/datascience/context/activeEditorContext.ts b/src/client/datascience/context/activeEditorContext.ts index 57b79eb8b79b..4950bca92446 100644 --- a/src/client/datascience/context/activeEditorContext.ts +++ b/src/client/datascience/context/activeEditorContext.ts @@ -62,6 +62,11 @@ export class ActiveEditorContextService implements IExtensionSingleActivationSer this, this.disposables ); + + // Do we already have python file opened. + if (this.docManager.activeTextEditor?.document.languageId === PYTHON_LANGUAGE) { + this.onDidChangeActiveTextEditor(this.docManager.activeTextEditor); + } } private onDidChangeActiveInteractiveWindow(e?: IInteractiveWindow) { From 2d0fef2f2df63083cdf708a51a52bd1f0e7cee5a Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Wed, 15 Apr 2020 15:57:08 -0700 Subject: [PATCH 029/725] Hide progress indicator once interactive window has loaded (#11143) * Hide progress when UI loads * News * Address code review --- news/2 Fixes/11065.md | 1 + .../interactive-window/interactiveWindow.ts | 11 +++++++++++ src/datascience-ui/history-react/interactivePanel.tsx | 2 +- .../history-react/redux/reducers/creation.ts | 11 +++++++++++ .../history-react/redux/reducers/index.ts | 1 + src/datascience-ui/native-editor/nativeEditor.tsx | 2 +- 6 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 news/2 Fixes/11065.md diff --git a/news/2 Fixes/11065.md b/news/2 Fixes/11065.md new file mode 100644 index 000000000000..fda10f86fcb1 --- /dev/null +++ b/news/2 Fixes/11065.md @@ -0,0 +1 @@ +Hide progress indicator once `Interactive Window` has loaded. diff --git a/src/client/datascience/interactive-window/interactiveWindow.ts b/src/client/datascience/interactive-window/interactiveWindow.ts index 5f59305a708a..5471bf75b113 100644 --- a/src/client/datascience/interactive-window/interactiveWindow.ts +++ b/src/client/datascience/interactive-window/interactiveWindow.ts @@ -274,6 +274,17 @@ export class InteractiveWindow extends InteractiveBase implements IInteractiveWi } return undefined; } + protected async addSysInfo(reason: SysInfoReason): Promise { + await super.addSysInfo(reason); + + // If `reason == Start`, then this means UI has been updated with the last + // pience of informaiotn (which was sys info), and now UI can be deemed as having been loaded. + // Marking a UI as having been loaded is done by sending a message `LoadAllCells`, even though we're not loading any cells. + // We're merely using existing messages (from NativeEditor). + if (reason === SysInfoReason.Start) { + this.postMessage(InteractiveWindowMessages.LoadAllCells, { cells: [] }).ignoreErrors(); + } + } protected async onViewStateChanged(args: WebViewViewChangeEventArgs) { super.onViewStateChanged(args); this._onDidChangeViewState.fire(); diff --git a/src/datascience-ui/history-react/interactivePanel.tsx b/src/datascience-ui/history-react/interactivePanel.tsx index be5f058c3c78..5beebc086dd0 100644 --- a/src/datascience-ui/history-react/interactivePanel.tsx +++ b/src/datascience-ui/history-react/interactivePanel.tsx @@ -51,7 +51,7 @@ export class InteractivePanel extends React.Component { fontFamily: this.props.font.family }; - const progressBar = this.props.busy && !this.props.testMode ? : undefined; + const progressBar = (this.props.busy || !this.props.loaded) && !this.props.testMode ? : undefined; // If in test mode, update our count. Use this to determine how many renders a normal update takes. if (this.props.testMode) { diff --git a/src/datascience-ui/history-react/redux/reducers/creation.ts b/src/datascience-ui/history-react/redux/reducers/creation.ts index a824b3c8a3f0..061c0b711cdd 100644 --- a/src/datascience-ui/history-react/redux/reducers/creation.ts +++ b/src/datascience-ui/history-react/redux/reducers/creation.ts @@ -183,4 +183,15 @@ export namespace Creation { editCellVM: undefined }; } + + export function loaded(arg: InteractiveReducerArg<{ cells: ICell[] }>): IMainState { + postActionToExtension(arg, InteractiveWindowMessages.LoadAllCellsComplete, { + cells: [] + }); + return { + ...arg.prevState, + loaded: true, + busy: false + }; + } } diff --git a/src/datascience-ui/history-react/redux/reducers/index.ts b/src/datascience-ui/history-react/redux/reducers/index.ts index 533397435765..6e661798ee00 100644 --- a/src/datascience-ui/history-react/redux/reducers/index.ts +++ b/src/datascience-ui/history-react/redux/reducers/index.ts @@ -34,6 +34,7 @@ export const reducerMap: Partial = { [CommonActionType.SUBMIT_INPUT]: Execution.submitInput, [InteractiveWindowMessages.ExpandAll]: Effects.expandAll, [CommonActionType.EDITOR_LOADED]: Transfer.started, + [InteractiveWindowMessages.LoadAllCells]: Creation.loaded, [CommonActionType.SCROLL]: Effects.scrolled, [CommonActionType.CLICK_CELL]: Effects.clickCell, [CommonActionType.UNFOCUS_CELL]: Effects.unfocusCell, diff --git a/src/datascience-ui/native-editor/nativeEditor.tsx b/src/datascience-ui/native-editor/nativeEditor.tsx index 0f4b1979f8db..e7bfe9ea7ed2 100644 --- a/src/datascience-ui/native-editor/nativeEditor.tsx +++ b/src/datascience-ui/native-editor/nativeEditor.tsx @@ -79,7 +79,7 @@ export class NativeEditor extends React.Component { } // Update the state controller with our new state - const progressBar = this.props.busy && !this.props.testMode ? : undefined; + const progressBar = (this.props.busy || !this.props.loaded) && !this.props.testMode ? : undefined; const addCellLine = this.props.cellVMs.length === 0 ? null : ( Date: Wed, 15 Apr 2020 16:27:50 -0700 Subject: [PATCH 030/725] Ds/greazer/fix gather webpack (#11109) * Support custom gather spec locations * Adjust gather->script generation * Interactive Window Gather * Created GatherToScript command and made the interactive window always use that when gathering. * Don't show gather if cell is error * No gathering markdown * Add a new folder to Gather spec path --- build/webpack/webpack.extension.config.js | 15 +++- package-lock.json | 8 +- package.json | 68 ++------------- src/client/common/types.ts | 8 +- src/client/datascience/gather/gather.ts | 82 +++++++++++++------ .../datascience/gather/gatherListener.ts | 58 +++++++++++-- .../interactiveWindowTypes.ts | 6 +- .../interactive-common/synchronization.ts | 4 +- .../history-react/interactiveCell.tsx | 8 +- .../history-react/redux/actions.ts | 2 + .../history-react/redux/reducers/index.ts | 1 + .../redux/reducers/transfer.ts | 10 ++- .../redux/reducers/types.ts | 2 + .../native-editor/nativeCell.tsx | 5 ++ .../native-editor/redux/actions.ts | 2 + .../native-editor/redux/reducers/index.ts | 1 + .../datascience/gather/gather.unit.test.ts | 47 +---------- .../interactivePanel.functional.test.tsx | 1 + .../interactiveWindow.functional.test.tsx | 34 ++++++-- .../specs/index.d.ts | 5 +- 20 files changed, 196 insertions(+), 171 deletions(-) diff --git a/build/webpack/webpack.extension.config.js b/build/webpack/webpack.extension.config.js index 63efdf641bd4..e51eaf12bc89 100644 --- a/build/webpack/webpack.extension.config.js +++ b/build/webpack/webpack.extension.config.js @@ -67,9 +67,18 @@ const config = { externals: ['vscode', 'commonjs', ...ppaPackageList, ...existingModulesInOutDir], plugins: [ ...common.getDefaultPlugins('extension'), - // Copy pdfkit after extension builds. webpack can't handle pdfkit. - new removeFilesWebpackPlugin({ - after: { include: ['./out/client/node_modules/pdfkit/js/pdfkit.standalone.*'] } + // Copy gather spec files into a known location for the webpacked extension + new FileManagerPlugin({ + onEnd: [ + { + copy: [ + { + source: './node_modules/@msrvida/python-program-analysis/dist/es5/specs/*.yaml', + destination: './out/client/gatherSpecs' + } + ] + } + ] }), new copyWebpackPlugin([ { diff --git a/package-lock.json b/package-lock.json index f22c8913328c..aad66a37a822 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4851,7 +4851,6 @@ "version": "1.0.10", "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dev": true, "requires": { "sprintf-js": "~1.0.2" } @@ -13720,7 +13719,6 @@ "version": "3.13.1", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz", "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==", - "dev": true, "requires": { "argparse": "^1.0.7", "esprima": "^4.0.0" @@ -13729,8 +13727,7 @@ "esprima": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "dev": true + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==" } } }, @@ -21222,8 +21219,7 @@ "sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=", - "dev": true + "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" }, "srcset": { "version": "1.0.0", diff --git a/package.json b/package.json index 32422ef55315..72219a305384 100644 --- a/package.json +++ b/package.json @@ -1655,68 +1655,6 @@ "markdownDescription": "Defines the location and order of the sources where scripts files for Widgets are downloaded from (e.g. ipywidgest, bqplot, beakerx, ipyleaflet, etc). Not selecting any of these could result in widgets not rendering or function correctly. See [here](https://aka.ms/PVSCIPyWidgets) for more information. Once updated you will need to restart the Kernel.", "scope": "machine" }, - "python.dataScience.gatherRules": { - "type": "array", - "default": [ - { - "objectName": "df", - "functionName": "head", - "doesNotModify": [ - "OBJECT" - ] - }, - { - "objectName": "df", - "functionName": "describe", - "doesNotModify": [ - "OBJECT" - ] - }, - { - "objectName": "df", - "functionName": "tail", - "doesNotModify": [ - "OBJECT" - ] - }, - { - "functionName": "print", - "doesNotModify": [ - "ARGUMENTS" - ] - }, - { - "functionName": "KMeans", - "doesNotModify": [ - "ARGUMENTS" - ] - }, - { - "functionName": "scatter", - "doesNotModify": [ - "ARGUMENTS" - ] - }, - { - "functionName": "fit", - "doesNotModify": [ - "ARGUMENTS" - ] - }, - { - "functionName": "sum", - "doesNotModify": [ - "ARGUMENTS" - ] - }, - { - "functionName": "len", - "doesNotModify": [ - "ARGUMENTS" - ] - } - ] - }, "python.dataScience.askForLargeDataFrames": { "type": "boolean", "default": true, @@ -1941,6 +1879,12 @@ "description": "Python Insiders Only: If experimental gather feature is enabled, gather code to a python script rather than a notebook.", "scope": "resource" }, + "python.dataScience.gatherSpecPath": { + "type": "string", + "default": "", + "description": "Python Insiders Only: If experimental gather feature is enabled, this setting specifies a folder that contains additional or replacement spec files used for analysis.", + "scope": "resource" + }, "python.dataScience.codeLenses": { "type": "string", "default": "python.datascience.runcell, python.datascience.runallcellsabove, python.datascience.debugcell", diff --git a/src/client/common/types.ts b/src/client/common/types.ts index 98b6f80d7e59..bce3dbe6ab23 100644 --- a/src/client/common/types.ts +++ b/src/client/common/types.ts @@ -327,12 +327,6 @@ export interface IAnalysisSettings { readonly logLevel: LogLevel; } -interface IGatherRule { - objectName?: string; - functionName: string; - doesNotModify: string[] | number[]; -} - export interface IVariableQuery { language: string; query: string; @@ -356,7 +350,7 @@ export interface IDataScienceSettings { maxOutputSize: number; enableGather?: boolean; gatherToScript?: boolean; - gatherRules?: IGatherRule[]; + gatherSpecPath?: string; sendSelectionToInteractiveWindow: boolean; markdownRegularExpression: string; codeRegularExpression: string; diff --git a/src/client/datascience/gather/gather.ts b/src/client/datascience/gather/gather.ts index 124fbd6f8ef1..ec14dad83b32 100644 --- a/src/client/datascience/gather/gather.ts +++ b/src/client/datascience/gather/gather.ts @@ -1,8 +1,10 @@ import * as ppatypes from '@msrvida-python-program-analysis'; import { inject, injectable } from 'inversify'; +import * as path from 'path'; import * as uuid from 'uuid/v4'; import { IApplicationShell, ICommandManager } from '../../common/application/types'; import { traceInfo } from '../../common/logger'; +import { IFileSystem } from '../../common/platform/types'; import { IConfigurationService, IDisposableRegistry } from '../../common/types'; import * as localize from '../../common/utils/localize'; // tslint:disable-next-line: no-duplicate-imports @@ -19,12 +21,14 @@ export class GatherProvider implements IGatherProvider { private _executionSlicer: ppatypes.ExecutionLogSlicer | undefined; private dataflowAnalyzer: ppatypes.DataflowAnalyzer | undefined; private _enabled: boolean; + private initPromise: Promise; constructor( @inject(IConfigurationService) private configService: IConfigurationService, @inject(IApplicationShell) private applicationShell: IApplicationShell, @inject(IDisposableRegistry) private disposables: IDisposableRegistry, - @inject(ICommandManager) private commandManager: ICommandManager + @inject(ICommandManager) private commandManager: ICommandManager, + @inject(IFileSystem) private fileSystem: IFileSystem ) { // Disable gather if we're not running on insiders. this._enabled = @@ -33,28 +37,12 @@ export class GatherProvider implements IGatherProvider { ? true : false; - if (this._enabled) { - try { - // tslint:disable-next-line: no-require-imports - const ppa = require('@msrvida/python-program-analysis') as typeof import('@msrvida-python-program-analysis'); - - if (ppa) { - this.dataflowAnalyzer = new ppa.DataflowAnalyzer(); - this._executionSlicer = new ppa.ExecutionLogSlicer(this.dataflowAnalyzer); - - this.disposables.push( - this.configService.getSettings(undefined).onDidChange((e) => this.updateEnableGather(e)) - ); - } - } catch (ex) { - traceInfo( - 'Gathering tools could not be activated. Indicates build of VSIX could not find @msrvida/python-program-analysis' - ); - } - } + this.initPromise = this.init(); } - public logExecution(vscCell: IVscCell): void { + public async logExecution(vscCell: IVscCell): Promise { + await this.initPromise; + const gatherCell = convertVscToGatherCell(vscCell); if (gatherCell) { @@ -65,6 +53,8 @@ export class GatherProvider implements IGatherProvider { } public async resetLog(): Promise { + await this.initPromise; + if (this._executionSlicer) { this._executionSlicer.reset(); } @@ -90,11 +80,7 @@ export class GatherProvider implements IGatherProvider { // Call internal slice method const slice = this._executionSlicer.sliceLatestExecution(gatherCell.persistentId); - const program = slice.cellSlices.reduce(concat, '').replace(/#%%/g, defaultCellMarker); - - // Add a comment at the top of the file explaining what gather does - const descriptor = localize.DataScience.gatheredScriptDescription(); - return descriptor.concat(program); + return slice.cellSlices.reduce(concat, '').replace(/#%%/g, defaultCellMarker); } public get executionSlicer() { @@ -124,6 +110,50 @@ export class GatherProvider implements IGatherProvider { } } } + + private async init(): Promise { + if (this._enabled) { + try { + // tslint:disable-next-line: no-require-imports + const ppa = require('@msrvida/python-program-analysis') as typeof import('@msrvida-python-program-analysis'); + + if (ppa) { + // If the __builtins__ specs are not available for gather, then no specs have been found. Look in a specific location relative + // to the extension. + if (!ppa.getSpecs()) { + const defaultSpecFolder = path.join(__dirname, 'gatherSpecs'); + if (await this.fileSystem.directoryExists(defaultSpecFolder)) { + ppa.setSpecFolder(defaultSpecFolder); + } + } + + // Check to see if any additional specs can be found. + const additionalSpecPath = this.configService.getSettings().datascience.gatherSpecPath; + if (additionalSpecPath && (await this.fileSystem.directoryExists(additionalSpecPath))) { + ppa.addSpecFolder(additionalSpecPath); + } else { + traceInfo(`Gather: additional spec folder ${additionalSpecPath} but not found.`); + } + + // Only continue to initialize gather if we were successful in finding SOME specs. + if (ppa.getSpecs()) { + this.dataflowAnalyzer = new ppa.DataflowAnalyzer(); + this._executionSlicer = new ppa.ExecutionLogSlicer(this.dataflowAnalyzer); + + this.disposables.push( + this.configService.getSettings(undefined).onDidChange((e) => this.updateEnableGather(e)) + ); + } else { + this._enabled = false; + traceInfo("Gather couldn't find any package specs. Disabling."); + } + } + } catch (ex) { + this._enabled = false; + traceInfo(`Gathering tools could't be activated. ${ex.message}`); + } + } + } } /** diff --git a/src/client/datascience/gather/gatherListener.ts b/src/client/datascience/gather/gatherListener.ts index 3226ed6036fd..019f92be522a 100644 --- a/src/client/datascience/gather/gatherListener.ts +++ b/src/client/datascience/gather/gatherListener.ts @@ -5,6 +5,7 @@ import { Event, EventEmitter, Position, Uri, ViewColumn } from 'vscode'; import { createMarkdownCell } from '../../../datascience-ui/common/cellFactory'; import { IApplicationShell, IDocumentManager } from '../../common/application/types'; import { PYTHON_LANGUAGE } from '../../common/constants'; +import { traceError } from '../../common/logger'; import { IFileSystem } from '../../common/platform/types'; import { IConfigurationService } from '../../common/types'; import * as localize from '../../common/utils/localize'; @@ -64,10 +65,14 @@ export class GatherListener implements IInteractiveWindowListener { this.handleMessage(message, payload, this.doInitGather); break; - case InteractiveWindowMessages.GatherCodeRequest: + case InteractiveWindowMessages.GatherCode: this.handleMessage(message, payload, this.doGather); break; + case InteractiveWindowMessages.GatherCodeToScript: + this.handleMessage(message, payload, this.doGatherToScript); + break; + case InteractiveWindowMessages.RestartKernel: if (this.gatherProvider) { this.gatherProvider.resetLog(); @@ -115,11 +120,19 @@ export class GatherListener implements IInteractiveWindowListener { private doGather(payload: ICell): void { this.gatherCodeInternal(payload).catch((err) => { + traceError(`Gather to Notebook error: ${err}`); this.applicationShell.showErrorMessage(err); }); } - private gatherCodeInternal = async (cell: ICell) => { + private doGatherToScript(payload: ICell): void { + this.gatherCodeInternal(payload, true).catch((err) => { + traceError(`Gather to Script error: ${err}`); + this.applicationShell.showErrorMessage(err); + }); + } + + private gatherCodeInternal = async (cell: ICell, toScript: boolean = false) => { this.gatherTimer = new StopWatch(); const slicedProgram = this.gatherProvider ? this.gatherProvider.gatherCode(cell) : 'Gather internal error'; @@ -127,7 +140,7 @@ export class GatherListener implements IInteractiveWindowListener { if (!slicedProgram) { sendTelemetryEvent(Telemetry.GatherCompleted, this.gatherTimer?.elapsedTime, { result: 'err' }); } else { - const gatherToScript: boolean | undefined = this.configService.getSettings().datascience.gatherToScript; + const gatherToScript: boolean = this.configService.getSettings().datascience.gatherToScript || toScript; if (gatherToScript) { await this.showFile(slicedProgram, cell.file); @@ -165,19 +178,46 @@ export class GatherListener implements IInteractiveWindowListener { const contents = JSON.stringify(notebook); const editor = await this.ipynbProvider.createNew(contents); - let disposable: IDisposable; - const handler = () => { + let disposableNotebookSaved: IDisposable; + let disposableNotebookClosed: IDisposable; + + const savedHandler = () => { sendTelemetryEvent(Telemetry.GatheredNotebookSaved); - if (disposable) { - disposable.dispose(); + if (disposableNotebookSaved) { + disposableNotebookSaved.dispose(); + } + if (disposableNotebookClosed) { + disposableNotebookClosed.dispose(); + } + }; + + const closedHandler = () => { + if (disposableNotebookSaved) { + disposableNotebookSaved.dispose(); + } + if (disposableNotebookClosed) { + disposableNotebookClosed.dispose(); } }; - disposable = editor.saved(handler); + + disposableNotebookSaved = editor.saved(savedHandler); + disposableNotebookClosed = editor.closed(closedHandler); } } } private async showFile(slicedProgram: string, filename: string) { + const defaultCellMarker = + this.configService.getSettings().datascience.defaultCellMarker || Identifiers.DefaultCodeCellMarker; + + if (slicedProgram) { + // Remove all cell definitions and newlines + const re = new RegExp(`^(${defaultCellMarker}.*|\\s*)\n`, 'gm'); + slicedProgram = slicedProgram.replace(re, ''); + } + + const annotatedScript = `${localize.DataScience.gatheredScriptDescription()}${defaultCellMarker}\n${slicedProgram}`; + // Don't want to open the gathered code on top of the interactive window let viewColumn: ViewColumn | undefined; const fileNameMatch = this.documentManager.visibleTextEditors.filter((textEditor) => @@ -199,7 +239,7 @@ export class GatherListener implements IInteractiveWindowListener { // Create a new open editor with the returned program in the right panel const doc = await this.documentManager.openTextDocument({ - content: slicedProgram, + content: annotatedScript, language: PYTHON_LANGUAGE }); const editor = await this.documentManager.showTextDocument(doc, viewColumn); diff --git a/src/client/datascience/interactive-common/interactiveWindowTypes.ts b/src/client/datascience/interactive-common/interactiveWindowTypes.ts index 49c1dc4c0ee7..df6b3090b96e 100644 --- a/src/client/datascience/interactive-common/interactiveWindowTypes.ts +++ b/src/client/datascience/interactive-common/interactiveWindowTypes.ts @@ -85,7 +85,8 @@ export enum InteractiveWindowMessages { SavePng = 'save_png', StartDebugging = 'start_debugging', StopDebugging = 'stop_debugging', - GatherCodeRequest = 'gather_code', + GatherCode = 'gather_code', + GatherCodeToScript = 'gather_code_to_script', LoadAllCells = 'load_all_cells', LoadAllCellsComplete = 'load_all_cells_complete', ScrollToCell = 'scroll_to_cell', @@ -572,7 +573,8 @@ export class IInteractiveWindowMapping { public [InteractiveWindowMessages.SavePng]: string | undefined; public [InteractiveWindowMessages.StartDebugging]: never | undefined; public [InteractiveWindowMessages.StopDebugging]: never | undefined; - public [InteractiveWindowMessages.GatherCodeRequest]: ICell; + public [InteractiveWindowMessages.GatherCode]: ICell; + public [InteractiveWindowMessages.GatherCodeToScript]: ICell; public [InteractiveWindowMessages.LoadAllCells]: ILoadAllCells; public [InteractiveWindowMessages.LoadAllCellsComplete]: ILoadAllCells; public [InteractiveWindowMessages.ScrollToCell]: IScrollToCell; diff --git a/src/client/datascience/interactive-common/synchronization.ts b/src/client/datascience/interactive-common/synchronization.ts index d26cfbeab21a..f0b96939f127 100644 --- a/src/client/datascience/interactive-common/synchronization.ts +++ b/src/client/datascience/interactive-common/synchronization.ts @@ -54,6 +54,7 @@ const messageWithMessageTypes: MessageMapping & Messa [CommonActionType.EXPORT]: MessageType.other, [CommonActionType.FOCUS_CELL]: MessageType.syncWithLiveShare, [CommonActionType.GATHER_CELL]: MessageType.other, + [CommonActionType.GATHER_CELL_TO_SCRIPT]: MessageType.other, [CommonActionType.GET_VARIABLE_DATA]: MessageType.other, [CommonActionType.GOTO_CELL]: MessageType.syncWithLiveShare, [CommonActionType.INSERT_ABOVE_AND_FOCUS_NEW_CELL]: MessageType.other, @@ -106,7 +107,8 @@ const messageWithMessageTypes: MessageMapping & Messa [InteractiveWindowMessages.Export]: MessageType.other, [InteractiveWindowMessages.FinishCell]: MessageType.syncAcrossSameNotebooks | MessageType.syncWithLiveShare, [InteractiveWindowMessages.FocusedCellEditor]: MessageType.syncWithLiveShare, - [InteractiveWindowMessages.GatherCodeRequest]: MessageType.other, + [InteractiveWindowMessages.GatherCode]: MessageType.other, + [InteractiveWindowMessages.GatherCodeToScript]: MessageType.other, [InteractiveWindowMessages.GetAllCells]: MessageType.other, [InteractiveWindowMessages.GetVariablesRequest]: MessageType.other, [InteractiveWindowMessages.GetVariablesResponse]: MessageType.other, diff --git a/src/datascience-ui/history-react/interactiveCell.tsx b/src/datascience-ui/history-react/interactiveCell.tsx index b4052c3f80d5..db70497643a3 100644 --- a/src/datascience-ui/history-react/interactiveCell.tsx +++ b/src/datascience-ui/history-react/interactiveCell.tsx @@ -175,7 +175,7 @@ export class InteractiveCell extends React.Component { const gotoCode = () => this.props.gotoCell(cellId); const deleteCode = () => this.props.deleteCell(cellId); const copyCode = () => this.props.copyCellCode(cellId); - const gatherCode = () => this.props.gatherCell(cellId); + const gatherCode = () => this.props.gatherCellToScript(cellId); const hasNoSource = !cell || !cell.file || cell.file === Identifiers.EmptyFileName; return ( @@ -183,7 +183,11 @@ export class InteractiveCell extends React.Component {