From 8ab5c9bf2d732a8168259be701431d05980e5a8a Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Tue, 23 Jun 2020 13:07:29 -0700 Subject: [PATCH 01/14] Move request header generation into password generation --- .../jupyter/jupyterPasswordConnect.ts | 20 +++++++++++++++---- .../jupyter/jupyterSessionManager.ts | 14 +++++-------- src/client/datascience/types.ts | 6 ++---- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/client/datascience/jupyter/jupyterPasswordConnect.ts b/src/client/datascience/jupyter/jupyterPasswordConnect.ts index 5a708081397f..13dce4754be3 100644 --- a/src/client/datascience/jupyter/jupyterPasswordConnect.ts +++ b/src/client/datascience/jupyter/jupyterPasswordConnect.ts @@ -43,11 +43,15 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { return result; } + private getSessionCookieString(xsrfCookie: string, sessionCookieName: string, sessionCookieValue: string): string { + return `_xsrf=${xsrfCookie}; ${sessionCookieName}=${sessionCookieValue}`; + } + private async getNonCachedPasswordConnectionInfo( url: string, allowUnauthorized: boolean, fetchFunction?: (url: nodeFetch.RequestInfo, init?: nodeFetch.RequestInit) => Promise - ) { + ): Promise { // For testing allow for our fetch function to be overridden if (!fetchFunction) { fetchFunction = nodeFetch.default; @@ -80,18 +84,20 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { } else { // If userPassword is undefined or '' then the user didn't pick a password. In this case return back that we should just try to connect // like a standard connection. Might be the case where there is no token and no password - return { emptyPassword: true, xsrfCookie: '', sessionCookieName: '', sessionCookieValue: '' }; + return {}; } userPassword = undefined; } else { // If no password needed, act like empty password and no cookie - return { emptyPassword: true, xsrfCookie: '', sessionCookieName: '', sessionCookieValue: '' }; + return {}; } // If we found everything return it all back if not, undefined as partial is useless if (xsrfCookie && sessionCookieName && sessionCookieValue) { sendTelemetryEvent(Telemetry.GetPasswordSuccess); - return { xsrfCookie, sessionCookieName, sessionCookieValue, emptyPassword: false }; + const cookieString = this.getSessionCookieString(xsrfCookie, sessionCookieName, sessionCookieValue); + const requestHeaders = { Cookie: cookieString, 'X-XSRFToken': xsrfCookie }; + return { requestHeaders }; } else { sendTelemetryEvent(Telemetry.GetPasswordFailure); return undefined; @@ -165,6 +171,12 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { return response.status !== 200; } + private async isJupyterHub(url: string): Promise { + // See this for the different REST endpoints: + // https://jupyterhub.readthedocs.io/en/stable/_static/rest-api/index.html + // Talk to either hub/api or check for a user token in the URI + } + // Jupyter uses a session cookie to validate so by hitting the login page with the password we can get that cookie and use it ourselves // This workflow can be seen by running fiddler and hitting the login page with a browser // First you need a get at the login page to get the xsrf token, then you send back that token along with the password in a post diff --git a/src/client/datascience/jupyter/jupyterSessionManager.ts b/src/client/datascience/jupyter/jupyterSessionManager.ts index 05342b6e3f67..cce062e6a634 100644 --- a/src/client/datascience/jupyter/jupyterSessionManager.ts +++ b/src/client/datascience/jupyter/jupyterSessionManager.ts @@ -218,10 +218,6 @@ export class JupyterSessionManager implements IJupyterSessionManager { } } - private getSessionCookieString(pwSettings: IJupyterPasswordConnectInfo): string { - return `_xsrf=${pwSettings.xsrfCookie}; ${pwSettings.sessionCookieName}=${pwSettings.sessionCookieValue}`; - } - private async getServerConnectSettings(connInfo: IJupyterConnection): Promise { let serverSettings: Partial = { baseUrl: connInfo.baseUrl, @@ -246,11 +242,11 @@ export class JupyterSessionManager implements IJupyterSessionManager { connInfo.baseUrl, connInfo.allowUnauthorized ? true : false ); - if (pwSettings && !pwSettings.emptyPassword) { - cookieString = this.getSessionCookieString(pwSettings); - const requestHeaders = { Cookie: cookieString, 'X-XSRFToken': pwSettings.xsrfCookie }; - requestInit = { ...requestInit, headers: requestHeaders }; - } else if (pwSettings && pwSettings.emptyPassword) { + if (pwSettings && pwSettings.requestHeaders) { + requestInit = { ...requestInit, headers: pwSettings.requestHeaders }; + // tslint:disable-next-line: no-any + cookieString = (pwSettings.requestHeaders as any).Cookie; + } else if (pwSettings) { serverSettings = { ...serverSettings, token: connInfo.token }; } else { // Failed to get password info, notify the user diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index 3b786b3d3a02..c7c3d9f0525a 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -306,10 +306,8 @@ export interface IJupyterDebugger { } export interface IJupyterPasswordConnectInfo { - emptyPassword: boolean; - xsrfCookie: string; - sessionCookieName: string; - sessionCookieValue: string; + requestHeaders?: HeadersInit; + remappedUrl?: string; } export const IJupyterPasswordConnect = Symbol('IJupyterPasswordConnect'); From 701957e130366b8a3770a7ba7409dea527d7a68d Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Tue, 23 Jun 2020 15:08:18 -0700 Subject: [PATCH 02/14] Connecting to token api --- package.nls.json | 1 + src/client/common/utils/localize.ts | 4 + .../jupyter/jupyterPasswordConnect.ts | 133 +++++++++++++----- .../jupyter/jupyterSessionManager.ts | 13 +- src/client/datascience/types.ts | 3 +- .../jupyterPasswordConnect.unit.test.ts | 26 +++- 6 files changed, 138 insertions(+), 42 deletions(-) diff --git a/package.nls.json b/package.nls.json index f3ce4c1222e7..79bfb8a67455 100644 --- a/package.nls.json +++ b/package.nls.json @@ -240,6 +240,7 @@ "DataScience.jupyterInstall": "Install", "DataScience.jupyterSelectURIPrompt": "Enter the URI of the running Jupyter server", "DataScience.jupyterSelectURIInvalidURI": "Invalid URI specified", + "DataScience.jupyterSelectUserPrompt": "Enter your notebook user", "DataScience.jupyterSelectPasswordPrompt": "Enter your notebook password", "DataScience.jupyterNotebookFailure": "Jupyter notebook failed to launch. \r\n{0}", "DataScience.jupyterNotebookConnectFailed": "Failed to connect to Jupyter notebook. \r\n{0}\r\n{1}", diff --git a/src/client/common/utils/localize.ts b/src/client/common/utils/localize.ts index b732fc126c4a..5de682c7be21 100644 --- a/src/client/common/utils/localize.ts +++ b/src/client/common/utils/localize.ts @@ -520,6 +520,10 @@ export namespace DataScience { 'DataScience.jupyterSelectURINotRunningDetail', 'Cannot connect at this time. Status unknown.' ); + export const jupyterSelectUserPrompt = localize( + 'DataScience.jupyterSelectUserPrompt', + 'Enter your notebook user name' + ); export const jupyterSelectPasswordPrompt = localize( 'DataScience.jupyterSelectPasswordPrompt', 'Enter your notebook password' diff --git a/src/client/datascience/jupyter/jupyterPasswordConnect.ts b/src/client/datascience/jupyter/jupyterPasswordConnect.ts index 13dce4754be3..19bd3da170db 100644 --- a/src/client/datascience/jupyter/jupyterPasswordConnect.ts +++ b/src/client/datascience/jupyter/jupyterPasswordConnect.ts @@ -14,6 +14,8 @@ import { Telemetry } from './../constants'; @injectable() export class JupyterPasswordConnect implements IJupyterPasswordConnect { private savedConnectInfo = new Map>(); + private fetchFunction: (url: nodeFetch.RequestInfo, init?: nodeFetch.RequestInit) => Promise = + nodeFetch.default; constructor(@inject(IApplicationShell) private appShell: IApplicationShell) {} @@ -27,6 +29,11 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { return Promise.resolve(undefined); } + // Update our fetch function if necessary + if (fetchFunction) { + this.fetchFunction = fetchFunction; + } + // Add on a trailing slash to our URL if it's not there already let newUrl = url; if (newUrl[newUrl.length - 1] !== '/') { @@ -36,7 +43,7 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { // See if we already have this data. Don't need to ask for a password more than once. (This can happen in remote when listing kernels) let result = this.savedConnectInfo.get(newUrl); if (!result) { - result = this.getNonCachedPasswordConnectionInfo(newUrl, allowUnauthorized, fetchFunction); + result = this.getNonCachedPasswordConnectionInfo(newUrl, allowUnauthorized); this.savedConnectInfo.set(newUrl, result); } @@ -49,35 +56,80 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { private async getNonCachedPasswordConnectionInfo( url: string, - allowUnauthorized: boolean, - fetchFunction?: (url: nodeFetch.RequestInfo, init?: nodeFetch.RequestInit) => Promise + allowUnauthorized: boolean ): Promise { - // For testing allow for our fetch function to be overridden - if (!fetchFunction) { - fetchFunction = nodeFetch.default; + // If jupyter hub, go down a special path of asking jupyter hub for a token + if (await this.isJupyterHub(url, allowUnauthorized)) { + return this.getJupyterHubConnectionInfo(url, allowUnauthorized); + } else { + return this.getJupyterConnectionInfo(url, allowUnauthorized); + } + } + + private async getJupyterHubConnectionInfo( + uri: string, + allowUnauthorized: boolean + ): Promise { + // We're using jupyter hub. Get the base url + let url: URL; + try { + url = new URL(uri); + } catch (err) { + // This should already have been parsed when set, so just throw if it's not right here + throw err; + } + const baseUrl = `${url.protocol}//${url.host}`; + + // First ask for the user name and password + const user = await this.getUserName(); + const password = user ? await this.getUserPassword() : ''; + + // Use these in a post request to get the token to use + const response = await this.fetchFunction( + `${baseUrl}/hub/api/authorizations/token`, // This seems to be deprecated, but it works. It requests a new token + this.addAllowUnauthorized(baseUrl, allowUnauthorized, { + method: 'post', + headers: { + Connection: 'keep-alive', + 'content-type': 'application/json;charset=UTF-8' + }, + body: `{ "username": "${user || ''}", "password": "${password || ''}" }`, + redirect: 'manual' + }) + ); + + if (response.ok && response.status === 200) { + const body = await response.json(); + if (body && body.user && body.user.server && body.token) { + // Response should have the token to use for this user. + return { + requestHeaders: {}, + remappedBaseUrl: `${baseUrl}/hub/api${body.user.server}`, + remappedToken: body.token + }; + } } + } + private async getJupyterConnectionInfo( + url: string, + allowUnauthorized: boolean + ): Promise { let xsrfCookie: string | undefined; let sessionCookieName: string | undefined; let sessionCookieValue: string | undefined; // First determine if we need a password. A request for the base URL with /tree? should return a 302 if we do. - if (await this.needPassword(url, allowUnauthorized, fetchFunction)) { + if (await this.needPassword(url, allowUnauthorized)) { // Get password first let userPassword = await this.getUserPassword(); if (userPassword) { - xsrfCookie = await this.getXSRFToken(url, allowUnauthorized, fetchFunction); + xsrfCookie = await this.getXSRFToken(url, allowUnauthorized); // Then get the session cookie by hitting that same page with the xsrftoken and the password if (xsrfCookie) { - const sessionResult = await this.getSessionCookie( - url, - allowUnauthorized, - xsrfCookie, - userPassword, - fetchFunction - ); + const sessionResult = await this.getSessionCookie(url, allowUnauthorized, xsrfCookie, userPassword); sessionCookieName = sessionResult.sessionCookieName; sessionCookieValue = sessionResult.sessionCookieValue; } @@ -118,8 +170,15 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { return options; } + private async getUserName(): Promise { + return this.appShell.showInputBox({ + prompt: localize.DataScience.jupyterSelectUserPrompt(), + ignoreFocusOut: true, + password: false + }); + } + private async getUserPassword(): Promise { - // First get the proposed URI from the user return this.appShell.showInputBox({ prompt: localize.DataScience.jupyterSelectPasswordPrompt(), ignoreFocusOut: true, @@ -127,14 +186,10 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { }); } - private async getXSRFToken( - url: string, - allowUnauthorized: boolean, - fetchFunction: (url: nodeFetch.RequestInfo, init?: nodeFetch.RequestInit) => Promise - ): Promise { + private async getXSRFToken(url: string, allowUnauthorized: boolean): Promise { let xsrfCookie: string | undefined; - const response = await fetchFunction( + const response = await this.fetchFunction( `${url}login?`, this.addAllowUnauthorized(url, allowUnauthorized, { method: 'get', @@ -153,13 +208,9 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { return xsrfCookie; } - private async needPassword( - url: string, - allowUnauthorized: boolean, - fetchFunction: (url: nodeFetch.RequestInfo, init?: nodeFetch.RequestInit) => Promise - ): Promise { + private async needPassword(url: string, allowUnauthorized: boolean): Promise { // A jupyter server will redirect if you ask for the tree when a login is required - const response = await fetchFunction( + const response = await this.fetchFunction( `${url}tree?`, this.addAllowUnauthorized(url, allowUnauthorized, { method: 'get', @@ -171,10 +222,27 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { return response.status !== 200; } - private async isJupyterHub(url: string): Promise { + private async isJupyterHub(url: string, allowUnauthorized: boolean): Promise { // See this for the different REST endpoints: // https://jupyterhub.readthedocs.io/en/stable/_static/rest-api/index.html - // Talk to either hub/api or check for a user token in the URI + + // If the URL has the /user/ option in it, it's likely this is jupyter hub + if (url.toLowerCase().includes('/user/')) { + return true; + } + + // Otherwise request hub/api. This should return the json with the hub version + // if this is a hub url + const response = await this.fetchFunction( + `${url}hub/api`, + this.addAllowUnauthorized(url, allowUnauthorized, { + method: 'get', + redirect: 'manual', + headers: { Connection: 'keep-alive' } + }) + ); + + return response.status === 200; } // Jupyter uses a session cookie to validate so by hitting the login page with the password we can get that cookie and use it ourselves @@ -185,8 +253,7 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { url: string, allowUnauthorized: boolean, xsrfCookie: string, - password: string, - fetchFunction: (url: nodeFetch.RequestInfo, init?: nodeFetch.RequestInit) => Promise + password: string ): Promise<{ sessionCookieName: string | undefined; sessionCookieValue: string | undefined }> { let sessionCookieName: string | undefined; let sessionCookieValue: string | undefined; @@ -195,7 +262,7 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { postParams.append('_xsrf', xsrfCookie); postParams.append('password', password); - const response = await fetchFunction( + const response = await this.fetchFunction( `${url}login?`, this.addAllowUnauthorized(url, allowUnauthorized, { method: 'post', diff --git a/src/client/datascience/jupyter/jupyterSessionManager.ts b/src/client/datascience/jupyter/jupyterSessionManager.ts index cce062e6a634..a61449563352 100644 --- a/src/client/datascience/jupyter/jupyterSessionManager.ts +++ b/src/client/datascience/jupyter/jupyterSessionManager.ts @@ -16,7 +16,6 @@ import { IJupyterKernel, IJupyterKernelSpec, IJupyterPasswordConnect, - IJupyterPasswordConnectInfo, IJupyterSession, IJupyterSessionManager } from '../types'; @@ -27,6 +26,8 @@ import { JupyterKernelSpec } from './kernels/jupyterKernelSpec'; import { KernelSelector } from './kernels/kernelSelector'; import { LiveKernelModel } from './kernels/types'; +// tslint:disable: no-any + export class JupyterSessionManager implements IJupyterSessionManager { private sessionManager: SessionManager | undefined; private contentsManager: ContentsManager | undefined; @@ -244,8 +245,16 @@ export class JupyterSessionManager implements IJupyterSessionManager { ); if (pwSettings && pwSettings.requestHeaders) { requestInit = { ...requestInit, headers: pwSettings.requestHeaders }; - // tslint:disable-next-line: no-any cookieString = (pwSettings.requestHeaders as any).Cookie; + + // Password may have overwritten the base url and token as well + if (pwSettings.remappedBaseUrl) { + (serverSettings as any).baseUrl = pwSettings.remappedBaseUrl; + (serverSettings as any).wsUrl = pwSettings.remappedBaseUrl.replace('http', 'ws'); + } + if (pwSettings.remappedToken) { + (serverSettings as any).token = pwSettings.remappedToken; + } } else if (pwSettings) { serverSettings = { ...serverSettings, token: connInfo.token }; } else { diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index c7c3d9f0525a..71ce472ffd12 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -307,7 +307,8 @@ export interface IJupyterDebugger { export interface IJupyterPasswordConnectInfo { requestHeaders?: HeadersInit; - remappedUrl?: string; + remappedBaseUrl?: string; + remappedToken?: string; } export const IJupyterPasswordConnect = Symbol('IJupyterPasswordConnect'); diff --git a/src/test/datascience/jupyterPasswordConnect.unit.test.ts b/src/test/datascience/jupyterPasswordConnect.unit.test.ts index 27627edddf41..c071ce678bb7 100644 --- a/src/test/datascience/jupyterPasswordConnect.unit.test.ts +++ b/src/test/datascience/jupyterPasswordConnect.unit.test.ts @@ -39,6 +39,10 @@ suite('JupyterPasswordConnect', () => { mockXsrfResponse.setup((mr) => mr.status).returns(() => 302); mockXsrfResponse.setup((mr) => mr.headers).returns(() => mockXsrfHeaders.object); + const mockHubResponse = typemoq.Mock.ofType(nodeFetch.Response); + mockHubResponse.setup((mr) => mr.ok).returns(() => false); + mockHubResponse.setup((mr) => mr.status).returns(() => 404); + fetchMock .setup((fm) => fm( @@ -62,6 +66,18 @@ suite('JupyterPasswordConnect', () => { ) ) .returns(() => Promise.resolve(mockXsrfResponse.object)); + fetchMock + .setup((fm) => + //tslint:disable-next-line:no-http-string + fm( + `${rootUrl}hub/api`, + typemoq.It.isObjectWith({ + method: 'get', + headers: { Connection: 'keep-alive' } + }) + ) + ) + .returns(() => Promise.resolve(mockHubResponse.object)); return { fetchMock, mockXsrfHeaders, mockXsrfResponse }; } @@ -102,9 +118,8 @@ suite('JupyterPasswordConnect', () => { ); assert(result, 'Failed to get password'); if (result) { - assert(result.xsrfCookie === xsrfValue, 'Incorrect xsrf value'); - assert(result.sessionCookieName === sessionName, 'Incorrect session name'); - assert(result.sessionCookieValue === sessionValue, 'Incorrect session value'); + // tslint:disable-next-line: no-cookies + assert.ok((result.requestHeaders as any).cookie, 'No cookie'); } // Verfiy calls @@ -154,9 +169,8 @@ suite('JupyterPasswordConnect', () => { ); assert(result, 'Failed to get password'); if (result) { - assert(result.xsrfCookie === xsrfValue, 'Incorrect xsrf value'); - assert(result.sessionCookieName === sessionName, 'Incorrect session name'); - assert(result.sessionCookieValue === sessionValue, 'Incorrect session value'); + // tslint:disable-next-line: no-cookies + assert.ok((result.requestHeaders as any).cookie, 'No cookie'); } // Verfiy calls From 4e5d536d117c1dda53763c58457129c4945bb7bd Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Tue, 23 Jun 2020 15:28:21 -0700 Subject: [PATCH 03/14] Working jupyter hub connection --- src/client/datascience/jupyter/jupyterPasswordConnect.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/datascience/jupyter/jupyterPasswordConnect.ts b/src/client/datascience/jupyter/jupyterPasswordConnect.ts index 19bd3da170db..3b5cedf331b5 100644 --- a/src/client/datascience/jupyter/jupyterPasswordConnect.ts +++ b/src/client/datascience/jupyter/jupyterPasswordConnect.ts @@ -104,7 +104,7 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { // Response should have the token to use for this user. return { requestHeaders: {}, - remappedBaseUrl: `${baseUrl}/hub/api${body.user.server}`, + remappedBaseUrl: `${baseUrl}${body.user.server}`, remappedToken: body.token }; } From ba44041d6de20c6e97b179cab0231c71fb9dfb0e Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Tue, 23 Jun 2020 16:05:43 -0700 Subject: [PATCH 04/14] Use multistep input --- package.nls.json | 5 +- src/client/common/utils/localize.ts | 9 +- src/client/common/utils/multiStepInput.ts | 3 + .../jupyter/jupyterPasswordConnect.ts | 97 +++++++++++++------ .../jupyterPasswordConnect.unit.test.ts | 5 +- 5 files changed, 81 insertions(+), 38 deletions(-) diff --git a/package.nls.json b/package.nls.json index 79bfb8a67455..133f74e65090 100644 --- a/package.nls.json +++ b/package.nls.json @@ -240,8 +240,9 @@ "DataScience.jupyterInstall": "Install", "DataScience.jupyterSelectURIPrompt": "Enter the URI of the running Jupyter server", "DataScience.jupyterSelectURIInvalidURI": "Invalid URI specified", - "DataScience.jupyterSelectUserPrompt": "Enter your notebook user", - "DataScience.jupyterSelectPasswordPrompt": "Enter your notebook password", + "DataScience.jupyterSelectUserAndPasswordTitle": "Enter your user name and password to connect to jupyter hub", + "DataScience.jupyterSelectUserPrompt": "Enter your user name", + "DataScience.jupyterSelectPasswordPrompt": "Enter your password", "DataScience.jupyterNotebookFailure": "Jupyter notebook failed to launch. \r\n{0}", "DataScience.jupyterNotebookConnectFailed": "Failed to connect to Jupyter notebook. \r\n{0}\r\n{1}", "DataScience.jupyterNotebookRemoteConnectFailed": "Failed to connect to remote Jupyter notebook.\r\nCheck that the Jupyter Server URI setting has a valid running server specified.\r\n{0}\r\n{1}", diff --git a/src/client/common/utils/localize.ts b/src/client/common/utils/localize.ts index 5de682c7be21..d7533a721ebb 100644 --- a/src/client/common/utils/localize.ts +++ b/src/client/common/utils/localize.ts @@ -520,13 +520,14 @@ export namespace DataScience { 'DataScience.jupyterSelectURINotRunningDetail', 'Cannot connect at this time. Status unknown.' ); - export const jupyterSelectUserPrompt = localize( - 'DataScience.jupyterSelectUserPrompt', - 'Enter your notebook user name' + export const jupyterSelectUserAndPasswordTitle = localize( + 'DataScience.jupyterSelectUserAndPasswordTitle', + 'Enter your user name and password to connect to jupyter hub' ); + export const jupyterSelectUserPrompt = localize('DataScience.jupyterSelectUserPrompt', 'Enter your user name'); export const jupyterSelectPasswordPrompt = localize( 'DataScience.jupyterSelectPasswordPrompt', - 'Enter your notebook password' + 'Enter your password' ); export const jupyterNotebookFailure = localize( 'DataScience.jupyterNotebookFailure', diff --git a/src/client/common/utils/multiStepInput.ts b/src/client/common/utils/multiStepInput.ts index fe10c7e8805f..03e68155e86a 100644 --- a/src/client/common/utils/multiStepInput.ts +++ b/src/client/common/utils/multiStepInput.ts @@ -39,6 +39,7 @@ export interface IQuickPickParameters { // tslint:disable-next-line: interface-name export interface InputBoxParameters { title: string; + password?: boolean; step?: number; totalSteps?: number; value: string; @@ -155,6 +156,7 @@ export class MultiStepInput implements IMultiStepInput { value, prompt, validate, + password, buttons, shouldResume }: P): Promise> { @@ -165,6 +167,7 @@ export class MultiStepInput implements IMultiStepInput { input.title = title; input.step = step; input.totalSteps = totalSteps; + input.password = password ? true : false; input.value = value || ''; input.prompt = prompt; input.ignoreFocusOut = true; diff --git a/src/client/datascience/jupyter/jupyterPasswordConnect.ts b/src/client/datascience/jupyter/jupyterPasswordConnect.ts index 3b5cedf331b5..4e86aae9212a 100644 --- a/src/client/datascience/jupyter/jupyterPasswordConnect.ts +++ b/src/client/datascience/jupyter/jupyterPasswordConnect.ts @@ -7,6 +7,7 @@ import * as nodeFetch from 'node-fetch'; import { URLSearchParams } from 'url'; import { IApplicationShell } from '../../common/application/types'; import * as localize from '../../common/utils/localize'; +import { IMultiStepInput, IMultiStepInputFactory } from '../../common/utils/multiStepInput'; import { captureTelemetry, sendTelemetryEvent } from '../../telemetry'; import { IJupyterPasswordConnect, IJupyterPasswordConnectInfo } from '../types'; import { Telemetry } from './../constants'; @@ -17,7 +18,10 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { private fetchFunction: (url: nodeFetch.RequestInfo, init?: nodeFetch.RequestInit) => Promise = nodeFetch.default; - constructor(@inject(IApplicationShell) private appShell: IApplicationShell) {} + constructor( + @inject(IApplicationShell) private appShell: IApplicationShell, + @inject(IMultiStepInputFactory) private readonly multiStepFactory: IMultiStepInputFactory + ) {} @captureTelemetry(Telemetry.GetPasswordAttempt) public getPasswordConnectionInfo( @@ -81,32 +85,32 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { const baseUrl = `${url.protocol}//${url.host}`; // First ask for the user name and password - const user = await this.getUserName(); - const password = user ? await this.getUserPassword() : ''; - - // Use these in a post request to get the token to use - const response = await this.fetchFunction( - `${baseUrl}/hub/api/authorizations/token`, // This seems to be deprecated, but it works. It requests a new token - this.addAllowUnauthorized(baseUrl, allowUnauthorized, { - method: 'post', - headers: { - Connection: 'keep-alive', - 'content-type': 'application/json;charset=UTF-8' - }, - body: `{ "username": "${user || ''}", "password": "${password || ''}" }`, - redirect: 'manual' - }) - ); - - if (response.ok && response.status === 200) { - const body = await response.json(); - if (body && body.user && body.user.server && body.token) { - // Response should have the token to use for this user. - return { - requestHeaders: {}, - remappedBaseUrl: `${baseUrl}${body.user.server}`, - remappedToken: body.token - }; + const result = await this.getUserNameAndPassword(); + if (result.username || result.password) { + // Use these in a post request to get the token to use + const response = await this.fetchFunction( + `${baseUrl}/hub/api/authorizations/token`, // This seems to be deprecated, but it works. It requests a new token + this.addAllowUnauthorized(baseUrl, allowUnauthorized, { + method: 'post', + headers: { + Connection: 'keep-alive', + 'content-type': 'application/json;charset=UTF-8' + }, + body: `{ "username": "${result.username || ''}", "password": "${result.password || ''}" }`, + redirect: 'manual' + }) + ); + + if (response.ok && response.status === 200) { + const body = await response.json(); + if (body && body.user && body.user.server && body.token) { + // Response should have the token to use for this user. + return { + requestHeaders: {}, + remappedBaseUrl: `${baseUrl}${body.user.server}`, + remappedToken: body.token + }; + } } } } @@ -170,11 +174,42 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { return options; } - private async getUserName(): Promise { - return this.appShell.showInputBox({ + private async getUserNameAndPassword(): Promise<{ username: string; password: string }> { + const multistep = this.multiStepFactory.create<{ username: string; password: string }>(); + const state = { username: '', password: '' }; + await multistep.run(this.getUserNameMultiStep.bind(this), state); + return state; + } + + private async getUserNameMultiStep( + input: IMultiStepInput<{ username: string; password: string }>, + state: { username: string; password: string } + ) { + state.username = await input.showInputBox({ + title: localize.DataScience.jupyterSelectUserAndPasswordTitle(), prompt: localize.DataScience.jupyterSelectUserPrompt(), - ignoreFocusOut: true, - password: false + validate: this.validateUserNameOrPassword, + value: '' + }); + if (state.username) { + return this.getPasswordMultiStep.bind(this); + } + } + + private async validateUserNameOrPassword(_value: string): Promise { + return undefined; + } + + private async getPasswordMultiStep( + input: IMultiStepInput<{ username: string; password: string }>, + state: { username: string; password: string } + ) { + state.password = await input.showInputBox({ + title: localize.DataScience.jupyterSelectUserAndPasswordTitle(), + prompt: localize.DataScience.jupyterSelectPasswordPrompt(), + validate: this.validateUserNameOrPassword, + value: '', + password: true }); } diff --git a/src/test/datascience/jupyterPasswordConnect.unit.test.ts b/src/test/datascience/jupyterPasswordConnect.unit.test.ts index c071ce678bb7..29e6f973ae9d 100644 --- a/src/test/datascience/jupyterPasswordConnect.unit.test.ts +++ b/src/test/datascience/jupyterPasswordConnect.unit.test.ts @@ -7,6 +7,7 @@ import * as nodeFetch from 'node-fetch'; import * as typemoq from 'typemoq'; import { IApplicationShell } from '../../client/common/application/types'; +import { MultiStepInputFactory } from '../../client/common/utils/multiStepInput'; import { JupyterPasswordConnect } from '../../client/datascience/jupyter/jupyterPasswordConnect'; // tslint:disable:no-any max-func-body-length @@ -21,7 +22,9 @@ suite('JupyterPasswordConnect', () => { setup(() => { appShell = typemoq.Mock.ofType(); appShell.setup((a) => a.showInputBox(typemoq.It.isAny())).returns(() => Promise.resolve('Python')); - jupyterPasswordConnect = new JupyterPasswordConnect(appShell.object); + const multiStepFactory = new MultiStepInputFactory(appShell.object); + + jupyterPasswordConnect = new JupyterPasswordConnect(appShell.object, multiStepFactory); }); function createMockSetup(secure: boolean, ok: boolean) { From 62185d86e04734a9adc244a0156c0a3b1063af4e Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Wed, 24 Jun 2020 13:27:38 -0700 Subject: [PATCH 05/14] Remove token on shutdown --- .../jupyter/jupyterPasswordConnect.ts | 162 ++++++++++++++++-- .../jupyterPasswordConnect.unit.test.ts | 9 +- 2 files changed, 154 insertions(+), 17 deletions(-) diff --git a/src/client/datascience/jupyter/jupyterPasswordConnect.ts b/src/client/datascience/jupyter/jupyterPasswordConnect.ts index 4e86aae9212a..5c4e5118bcc6 100644 --- a/src/client/datascience/jupyter/jupyterPasswordConnect.ts +++ b/src/client/datascience/jupyter/jupyterPasswordConnect.ts @@ -6,6 +6,7 @@ import { inject, injectable } from 'inversify'; import * as nodeFetch from 'node-fetch'; import { URLSearchParams } from 'url'; import { IApplicationShell } from '../../common/application/types'; +import { IAsyncDisposableRegistry } from '../../common/types'; import * as localize from '../../common/utils/localize'; import { IMultiStepInput, IMultiStepInputFactory } from '../../common/utils/multiStepInput'; import { captureTelemetry, sendTelemetryEvent } from '../../telemetry'; @@ -20,7 +21,8 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { constructor( @inject(IApplicationShell) private appShell: IApplicationShell, - @inject(IMultiStepInputFactory) private readonly multiStepFactory: IMultiStepInputFactory + @inject(IMultiStepInputFactory) private readonly multiStepFactory: IMultiStepInputFactory, + @inject(IAsyncDisposableRegistry) private readonly asyncDisposableRegistry: IAsyncDisposableRegistry ) {} @captureTelemetry(Telemetry.GetPasswordAttempt) @@ -73,6 +75,37 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { private async getJupyterHubConnectionInfo( uri: string, allowUnauthorized: boolean + ): Promise { + // First ask for the user name and password + const userNameAndPassword = await this.getUserNameAndPassword(); + if (userNameAndPassword.username || userNameAndPassword.password) { + // Try the login method. It should work and doesn't require a token to be generated. + const result = await this.getJupyterHubConnectionInfoFromLogin( + uri, + allowUnauthorized, + userNameAndPassword.username, + userNameAndPassword.password + ); + + // If login method fails, try generating a token + if (!result) { + return this.getJupyterHubConnectionInfoFromApi( + uri, + allowUnauthorized, + userNameAndPassword.username, + userNameAndPassword.password + ); + } + + return result; + } + } + + private async getJupyterHubConnectionInfoFromLogin( + uri: string, + allowUnauthorized: boolean, + username: string, + password: string ): Promise { // We're using jupyter hub. Get the base url let url: URL; @@ -84,30 +117,85 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { } const baseUrl = `${url.protocol}//${url.host}`; - // First ask for the user name and password - const result = await this.getUserNameAndPassword(); - if (result.username || result.password) { - // Use these in a post request to get the token to use - const response = await this.fetchFunction( - `${baseUrl}/hub/api/authorizations/token`, // This seems to be deprecated, but it works. It requests a new token + const postParams = new URLSearchParams(); + postParams.append('username', username || ''); + postParams.append('password', password || ''); + + let response = await this.fetchFunction( + `${baseUrl}/hub/login?next=`, + this.addAllowUnauthorized(baseUrl, allowUnauthorized, { + method: 'POST', + headers: { + Connection: 'keep-alive', + Referer: `${baseUrl}/hub/login`, + 'content-type': 'application/x-www-form-urlencoded;charset=UTF-8' + }, + body: postParams.toString(), + redirect: 'manual' + }) + ); + + // The cookies from that response should be used to make the next set of requests + if (response && response.status === 302) { + const cookies = this.getCookies(response); + const cookieString = [...cookies.entries()].reduce((p, c) => `${p};${c[0]}=${c[1]}`, ''); + // See this API for creating a token + // https://jupyterhub.readthedocs.io/en/stable/_static/rest-api/index.html#operation--users--name--tokens-post + response = await this.fetchFunction( + `${baseUrl}/hub/api/users/${username}/tokens`, this.addAllowUnauthorized(baseUrl, allowUnauthorized, { - method: 'post', + method: 'POST', headers: { Connection: 'keep-alive', - 'content-type': 'application/json;charset=UTF-8' - }, - body: `{ "username": "${result.username || ''}", "password": "${result.password || ''}" }`, - redirect: 'manual' + Cookie: cookieString, + Referer: `${baseUrl}/hub/login` + } }) ); + // That should give us a new token. For now server name is hard coded. Not sure + // how to fetch it other than in the info for a default token if (response.ok && response.status === 200) { const body = await response.json(); - if (body && body.user && body.user.server && body.token) { + if (body && body.token && body.id) { // Response should have the token to use for this user. + + // Make sure the server is running for this user. Don't need + // to check response as it will fail if already running. + // https://jupyterhub.readthedocs.io/en/stable/_static/rest-api/index.html#operation--users--name--server-post + await this.fetchFunction( + `${baseUrl}/hub/api/users/${username}/server`, + this.addAllowUnauthorized(baseUrl, allowUnauthorized, { + method: 'POST', + headers: { + Connection: 'keep-alive', + Cookie: cookieString, + Referer: `${baseUrl}/hub/login` + } + }) + ); + + // This token was generated for this request. We should clean it up when + // the user closes VS code + this.asyncDisposableRegistry.push({ + dispose: async () => { + await this.fetchFunction( + `${baseUrl}/hub/api/users/${username}/tokens/${body.id}`, + this.addAllowUnauthorized(baseUrl, allowUnauthorized, { + method: 'DELETE', + headers: { + Connection: 'keep-alive', + Cookie: cookieString, + Referer: `${baseUrl}/hub/login` + } + }) + ); + } + }); + return { requestHeaders: {}, - remappedBaseUrl: `${baseUrl}${body.user.server}`, + remappedBaseUrl: `${baseUrl}/user/${username}`, remappedToken: body.token }; } @@ -115,6 +203,48 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { } } + private async getJupyterHubConnectionInfoFromApi( + uri: string, + allowUnauthorized: boolean, + username: string, + password: string + ): Promise { + // We're using jupyter hub. Get the base url + let url: URL; + try { + url = new URL(uri); + } catch (err) { + // This should already have been parsed when set, so just throw if it's not right here + throw err; + } + const baseUrl = `${url.protocol}//${url.host}`; + // Use these in a post request to get the token to use + const response = await this.fetchFunction( + `${baseUrl}/hub/api/authorizations/token`, // This seems to be deprecated, but it works. It requests a new token + this.addAllowUnauthorized(baseUrl, allowUnauthorized, { + method: 'POST', + headers: { + Connection: 'keep-alive', + 'content-type': 'application/json;charset=UTF-8' + }, + body: `{ "username": "${username || ''}", "password": "${password || ''}" }`, + redirect: 'manual' + }) + ); + + if (response.ok && response.status === 200) { + const body = await response.json(); + if (body && body.user && body.user.server && body.token) { + // Response should have the token to use for this user. + return { + requestHeaders: {}, + remappedBaseUrl: `${baseUrl}${body.user.server}`, + remappedToken: body.token + }; + } + } + } + private async getJupyterConnectionInfo( url: string, allowUnauthorized: boolean @@ -328,10 +458,10 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { private getCookies(response: nodeFetch.Response): Map { const cookieList: Map = new Map(); - const cookies: string | null = response.headers.get('set-cookie'); + const cookies = response.headers.raw()['set-cookie']; if (cookies) { - cookies.split(';').forEach((value) => { + cookies.forEach((value) => { const cookieKey = value.substring(0, value.indexOf('=')); const cookieVal = value.substring(value.indexOf('=') + 1); cookieList.set(cookieKey, cookieVal); diff --git a/src/test/datascience/jupyterPasswordConnect.unit.test.ts b/src/test/datascience/jupyterPasswordConnect.unit.test.ts index 29e6f973ae9d..b2d669563c20 100644 --- a/src/test/datascience/jupyterPasswordConnect.unit.test.ts +++ b/src/test/datascience/jupyterPasswordConnect.unit.test.ts @@ -6,7 +6,9 @@ import * as assert from 'assert'; import * as nodeFetch from 'node-fetch'; import * as typemoq from 'typemoq'; +import { instance, mock } from 'ts-mockito'; import { IApplicationShell } from '../../client/common/application/types'; +import { AsyncDisposableRegistry } from '../../client/common/asyncDisposableRegistry'; import { MultiStepInputFactory } from '../../client/common/utils/multiStepInput'; import { JupyterPasswordConnect } from '../../client/datascience/jupyter/jupyterPasswordConnect'; @@ -23,8 +25,13 @@ suite('JupyterPasswordConnect', () => { appShell = typemoq.Mock.ofType(); appShell.setup((a) => a.showInputBox(typemoq.It.isAny())).returns(() => Promise.resolve('Python')); const multiStepFactory = new MultiStepInputFactory(appShell.object); + const mockDisposableRegistry = mock(AsyncDisposableRegistry); - jupyterPasswordConnect = new JupyterPasswordConnect(appShell.object, multiStepFactory); + jupyterPasswordConnect = new JupyterPasswordConnect( + appShell.object, + multiStepFactory, + instance(mockDisposableRegistry) + ); }); function createMockSetup(secure: boolean, ok: boolean) { From bf8dad4e32437e84e6336b290a62575e25fa00b3 Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Wed, 24 Jun 2020 16:48:18 -0700 Subject: [PATCH 06/14] Support certificate checking --- .../datascience/jupyter/jupyterExecution.ts | 2 +- .../jupyter/jupyterPasswordConnect.ts | 232 +++++++++--------- .../jupyter/jupyterSessionManager.ts | 14 +- .../jupyter/jupyterSessionManagerFactory.ts | 3 +- .../datascience/jupyter/jupyterUtils.ts | 7 +- src/client/datascience/types.ts | 6 +- .../jupyter/jupyterConnection.unit.test.ts | 1 - .../jupyterPasswordConnect.unit.test.ts | 8 +- 8 files changed, 131 insertions(+), 142 deletions(-) diff --git a/src/client/datascience/jupyter/jupyterExecution.ts b/src/client/datascience/jupyter/jupyterExecution.ts index cbfface0cf62..d4430a709c1e 100644 --- a/src/client/datascience/jupyter/jupyterExecution.ts +++ b/src/client/datascience/jupyter/jupyterExecution.ts @@ -363,7 +363,7 @@ export class JupyterExecutionBase implements IJupyterExecution { } } else { // If we have a URI spec up a connection info for it - return createRemoteConnectionInfo(options.uri, this.configuration.getSettings(undefined).datascience); + return createRemoteConnectionInfo(options.uri); } } diff --git a/src/client/datascience/jupyter/jupyterPasswordConnect.ts b/src/client/datascience/jupyter/jupyterPasswordConnect.ts index 5c4e5118bcc6..deda5376c53c 100644 --- a/src/client/datascience/jupyter/jupyterPasswordConnect.ts +++ b/src/client/datascience/jupyter/jupyterPasswordConnect.ts @@ -5,8 +5,9 @@ import { Agent as HttpsAgent } from 'https'; import { inject, injectable } from 'inversify'; import * as nodeFetch from 'node-fetch'; import { URLSearchParams } from 'url'; +import { ConfigurationTarget } from 'vscode'; import { IApplicationShell } from '../../common/application/types'; -import { IAsyncDisposableRegistry } from '../../common/types'; +import { IAsyncDisposableRegistry, IConfigurationService } from '../../common/types'; import * as localize from '../../common/utils/localize'; import { IMultiStepInput, IMultiStepInputFactory } from '../../common/utils/multiStepInput'; import { captureTelemetry, sendTelemetryEvent } from '../../telemetry'; @@ -22,13 +23,13 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { constructor( @inject(IApplicationShell) private appShell: IApplicationShell, @inject(IMultiStepInputFactory) private readonly multiStepFactory: IMultiStepInputFactory, - @inject(IAsyncDisposableRegistry) private readonly asyncDisposableRegistry: IAsyncDisposableRegistry + @inject(IAsyncDisposableRegistry) private readonly asyncDisposableRegistry: IAsyncDisposableRegistry, + @inject(IConfigurationService) private readonly configService: IConfigurationService ) {} @captureTelemetry(Telemetry.GetPasswordAttempt) public getPasswordConnectionInfo( url: string, - allowUnauthorized: boolean, fetchFunction?: (url: nodeFetch.RequestInfo, init?: nodeFetch.RequestInit) => Promise ): Promise { if (!url || url.length < 1) { @@ -49,7 +50,7 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { // See if we already have this data. Don't need to ask for a password more than once. (This can happen in remote when listing kernels) let result = this.savedConnectInfo.get(newUrl); if (!result) { - result = this.getNonCachedPasswordConnectionInfo(newUrl, allowUnauthorized); + result = this.getNonCachedPasswordConnectionInfo(newUrl); this.savedConnectInfo.set(newUrl, result); } @@ -60,29 +61,22 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { return `_xsrf=${xsrfCookie}; ${sessionCookieName}=${sessionCookieValue}`; } - private async getNonCachedPasswordConnectionInfo( - url: string, - allowUnauthorized: boolean - ): Promise { + private async getNonCachedPasswordConnectionInfo(url: string): Promise { // If jupyter hub, go down a special path of asking jupyter hub for a token - if (await this.isJupyterHub(url, allowUnauthorized)) { - return this.getJupyterHubConnectionInfo(url, allowUnauthorized); + if (await this.isJupyterHub(url)) { + return this.getJupyterHubConnectionInfo(url); } else { - return this.getJupyterConnectionInfo(url, allowUnauthorized); + return this.getJupyterConnectionInfo(url); } } - private async getJupyterHubConnectionInfo( - uri: string, - allowUnauthorized: boolean - ): Promise { + private async getJupyterHubConnectionInfo(uri: string): Promise { // First ask for the user name and password const userNameAndPassword = await this.getUserNameAndPassword(); if (userNameAndPassword.username || userNameAndPassword.password) { // Try the login method. It should work and doesn't require a token to be generated. const result = await this.getJupyterHubConnectionInfoFromLogin( uri, - allowUnauthorized, userNameAndPassword.username, userNameAndPassword.password ); @@ -91,7 +85,6 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { if (!result) { return this.getJupyterHubConnectionInfoFromApi( uri, - allowUnauthorized, userNameAndPassword.username, userNameAndPassword.password ); @@ -103,7 +96,6 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { private async getJupyterHubConnectionInfoFromLogin( uri: string, - allowUnauthorized: boolean, username: string, password: string ): Promise { @@ -121,19 +113,16 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { postParams.append('username', username || ''); postParams.append('password', password || ''); - let response = await this.fetchFunction( - `${baseUrl}/hub/login?next=`, - this.addAllowUnauthorized(baseUrl, allowUnauthorized, { - method: 'POST', - headers: { - Connection: 'keep-alive', - Referer: `${baseUrl}/hub/login`, - 'content-type': 'application/x-www-form-urlencoded;charset=UTF-8' - }, - body: postParams.toString(), - redirect: 'manual' - }) - ); + let response = await this.makeRequest(`${baseUrl}/hub/login?next=`, { + method: 'POST', + headers: { + Connection: 'keep-alive', + Referer: `${baseUrl}/hub/login`, + 'content-type': 'application/x-www-form-urlencoded;charset=UTF-8' + }, + body: postParams.toString(), + redirect: 'manual' + }); // The cookies from that response should be used to make the next set of requests if (response && response.status === 302) { @@ -141,17 +130,14 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { const cookieString = [...cookies.entries()].reduce((p, c) => `${p};${c[0]}=${c[1]}`, ''); // See this API for creating a token // https://jupyterhub.readthedocs.io/en/stable/_static/rest-api/index.html#operation--users--name--tokens-post - response = await this.fetchFunction( - `${baseUrl}/hub/api/users/${username}/tokens`, - this.addAllowUnauthorized(baseUrl, allowUnauthorized, { - method: 'POST', - headers: { - Connection: 'keep-alive', - Cookie: cookieString, - Referer: `${baseUrl}/hub/login` - } - }) - ); + response = await this.makeRequest(`${baseUrl}/hub/api/users/${username}/tokens`, { + method: 'POST', + headers: { + Connection: 'keep-alive', + Cookie: cookieString, + Referer: `${baseUrl}/hub/login` + } + }); // That should give us a new token. For now server name is hard coded. Not sure // how to fetch it other than in the info for a default token @@ -163,33 +149,27 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { // Make sure the server is running for this user. Don't need // to check response as it will fail if already running. // https://jupyterhub.readthedocs.io/en/stable/_static/rest-api/index.html#operation--users--name--server-post - await this.fetchFunction( - `${baseUrl}/hub/api/users/${username}/server`, - this.addAllowUnauthorized(baseUrl, allowUnauthorized, { - method: 'POST', - headers: { - Connection: 'keep-alive', - Cookie: cookieString, - Referer: `${baseUrl}/hub/login` - } - }) - ); + await this.makeRequest(`${baseUrl}/hub/api/users/${username}/server`, { + method: 'POST', + headers: { + Connection: 'keep-alive', + Cookie: cookieString, + Referer: `${baseUrl}/hub/login` + } + }); // This token was generated for this request. We should clean it up when // the user closes VS code this.asyncDisposableRegistry.push({ dispose: async () => { - await this.fetchFunction( - `${baseUrl}/hub/api/users/${username}/tokens/${body.id}`, - this.addAllowUnauthorized(baseUrl, allowUnauthorized, { - method: 'DELETE', - headers: { - Connection: 'keep-alive', - Cookie: cookieString, - Referer: `${baseUrl}/hub/login` - } - }) - ); + await this.makeRequest(`${baseUrl}/hub/api/users/${username}/tokens/${body.id}`, { + method: 'DELETE', + headers: { + Connection: 'keep-alive', + Cookie: cookieString, + Referer: `${baseUrl}/hub/login` + } + }); } }); @@ -205,7 +185,6 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { private async getJupyterHubConnectionInfoFromApi( uri: string, - allowUnauthorized: boolean, username: string, password: string ): Promise { @@ -219,9 +198,9 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { } const baseUrl = `${url.protocol}//${url.host}`; // Use these in a post request to get the token to use - const response = await this.fetchFunction( + const response = await this.makeRequest( `${baseUrl}/hub/api/authorizations/token`, // This seems to be deprecated, but it works. It requests a new token - this.addAllowUnauthorized(baseUrl, allowUnauthorized, { + { method: 'POST', headers: { Connection: 'keep-alive', @@ -229,7 +208,7 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { }, body: `{ "username": "${username || ''}", "password": "${password || ''}" }`, redirect: 'manual' - }) + } ); if (response.ok && response.status === 200) { @@ -245,25 +224,22 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { } } - private async getJupyterConnectionInfo( - url: string, - allowUnauthorized: boolean - ): Promise { + private async getJupyterConnectionInfo(url: string): Promise { let xsrfCookie: string | undefined; let sessionCookieName: string | undefined; let sessionCookieValue: string | undefined; // First determine if we need a password. A request for the base URL with /tree? should return a 302 if we do. - if (await this.needPassword(url, allowUnauthorized)) { + if (await this.needPassword(url)) { // Get password first let userPassword = await this.getUserPassword(); if (userPassword) { - xsrfCookie = await this.getXSRFToken(url, allowUnauthorized); + xsrfCookie = await this.getXSRFToken(url); // Then get the session cookie by hitting that same page with the xsrftoken and the password if (xsrfCookie) { - const sessionResult = await this.getSessionCookie(url, allowUnauthorized, xsrfCookie, userPassword); + const sessionResult = await this.getSessionCookie(url, xsrfCookie, userPassword); sessionCookieName = sessionResult.sessionCookieName; sessionCookieValue = sessionResult.sessionCookieValue; } @@ -351,17 +327,14 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { }); } - private async getXSRFToken(url: string, allowUnauthorized: boolean): Promise { + private async getXSRFToken(url: string): Promise { let xsrfCookie: string | undefined; - const response = await this.fetchFunction( - `${url}login?`, - this.addAllowUnauthorized(url, allowUnauthorized, { - method: 'get', - redirect: 'manual', - headers: { Connection: 'keep-alive' } - }) - ); + const response = await this.makeRequest(`${url}login?`, { + method: 'get', + redirect: 'manual', + headers: { Connection: 'keep-alive' } + }); if (response.ok) { const cookies = this.getCookies(response); @@ -373,21 +346,55 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { return xsrfCookie; } - private async needPassword(url: string, allowUnauthorized: boolean): Promise { + private async needPassword(url: string): Promise { // A jupyter server will redirect if you ask for the tree when a login is required - const response = await this.fetchFunction( - `${url}tree?`, - this.addAllowUnauthorized(url, allowUnauthorized, { - method: 'get', - redirect: 'manual', - headers: { Connection: 'keep-alive' } - }) - ); + const response = await this.makeRequest(`${url}tree?`, { + method: 'get', + redirect: 'manual', + headers: { Connection: 'keep-alive' } + }); return response.status !== 200; } - private async isJupyterHub(url: string, allowUnauthorized: boolean): Promise { + private async makeRequest(url: string, options: nodeFetch.RequestInit): Promise { + const allowUnauthorized = this.configService.getSettings(undefined).datascience + .allowUnauthorizedRemoteConnection; + + // Try once and see if it fails with unauthorized. + try { + return await this.fetchFunction( + url, + this.addAllowUnauthorized(url, allowUnauthorized ? true : false, options) + ); + } catch (e) { + if (e.message.indexOf('reason: self signed certificate') >= 0) { + // Ask user to change setting and possibly try again. + const enableOption: string = localize.DataScience.jupyterSelfCertEnable(); + const closeOption: string = localize.DataScience.jupyterSelfCertClose(); + const value = await this.appShell.showErrorMessage( + localize.DataScience.jupyterSelfCertFail().format(e.message), + enableOption, + closeOption + ); + if (value === enableOption) { + sendTelemetryEvent(Telemetry.SelfCertsMessageEnabled); + await this.configService.updateSetting( + 'dataScience.allowUnauthorizedRemoteConnection', + true, + undefined, + ConfigurationTarget.Workspace + ); + return this.fetchFunction(url, this.addAllowUnauthorized(url, true, options)); + } else if (value === closeOption) { + sendTelemetryEvent(Telemetry.SelfCertsMessageClose); + } + } + throw e; + } + } + + private async isJupyterHub(url: string): Promise { // See this for the different REST endpoints: // https://jupyterhub.readthedocs.io/en/stable/_static/rest-api/index.html @@ -398,14 +405,11 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { // Otherwise request hub/api. This should return the json with the hub version // if this is a hub url - const response = await this.fetchFunction( - `${url}hub/api`, - this.addAllowUnauthorized(url, allowUnauthorized, { - method: 'get', - redirect: 'manual', - headers: { Connection: 'keep-alive' } - }) - ); + const response = await this.makeRequest(`${url}hub/api`, { + method: 'get', + redirect: 'manual', + headers: { Connection: 'keep-alive' } + }); return response.status === 200; } @@ -416,7 +420,6 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { // That will return back the session cookie. This session cookie then needs to be added to our requests and websockets for @jupyterlab/services private async getSessionCookie( url: string, - allowUnauthorized: boolean, xsrfCookie: string, password: string ): Promise<{ sessionCookieName: string | undefined; sessionCookieValue: string | undefined }> { @@ -427,19 +430,16 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { postParams.append('_xsrf', xsrfCookie); postParams.append('password', password); - const response = await this.fetchFunction( - `${url}login?`, - this.addAllowUnauthorized(url, allowUnauthorized, { - method: 'post', - headers: { - Cookie: `_xsrf=${xsrfCookie}`, - Connection: 'keep-alive', - 'content-type': 'application/x-www-form-urlencoded;charset=UTF-8' - }, - body: postParams.toString(), - redirect: 'manual' - }) - ); + const response = await this.makeRequest(`${url}login?`, { + method: 'post', + headers: { + Cookie: `_xsrf=${xsrfCookie}`, + Connection: 'keep-alive', + 'content-type': 'application/x-www-form-urlencoded;charset=UTF-8' + }, + body: postParams.toString(), + redirect: 'manual' + }); // Now from this result we need to extract the session cookie if (response.status === 302) { diff --git a/src/client/datascience/jupyter/jupyterSessionManager.ts b/src/client/datascience/jupyter/jupyterSessionManager.ts index a61449563352..6a0816e70c31 100644 --- a/src/client/datascience/jupyter/jupyterSessionManager.ts +++ b/src/client/datascience/jupyter/jupyterSessionManager.ts @@ -46,7 +46,8 @@ export class JupyterSessionManager implements IJupyterSessionManager { _config: IConfigurationService, private failOnPassword: boolean | undefined, private kernelSelector: KernelSelector, - private outputChannel: IOutputChannel + private outputChannel: IOutputChannel, + private configService: IConfigurationService ) {} public async dispose() { @@ -231,7 +232,6 @@ export class JupyterSessionManager implements IJupyterSessionManager { // tslint:disable-next-line:no-any let requestInit: any = { cache: 'no-store', credentials: 'same-origin' }; let cookieString; - let allowUnauthorized; // If no token is specified prompt for a password if (connInfo.token === '' || connInfo.token === 'null') { @@ -239,10 +239,7 @@ export class JupyterSessionManager implements IJupyterSessionManager { throw new Error('Password request not allowed.'); } serverSettings = { ...serverSettings, token: '' }; - const pwSettings = await this.jupyterPasswordConnect.getPasswordConnectionInfo( - connInfo.baseUrl, - connInfo.allowUnauthorized ? true : false - ); + const pwSettings = await this.jupyterPasswordConnect.getPasswordConnectionInfo(connInfo.baseUrl); if (pwSettings && pwSettings.requestHeaders) { requestInit = { ...requestInit, headers: pwSettings.requestHeaders }; cookieString = (pwSettings.requestHeaders as any).Cookie; @@ -265,12 +262,13 @@ export class JupyterSessionManager implements IJupyterSessionManager { serverSettings = { ...serverSettings, token: connInfo.token }; } + const allowUnauthorized = this.configService.getSettings(undefined).datascience + .allowUnauthorizedRemoteConnection; // If this is an https connection and we want to allow unauthorized connections set that option on our agent // we don't need to save the agent as the previous behaviour is just to create a temporary default agent when not specified - if (connInfo.baseUrl.startsWith('https') && connInfo.allowUnauthorized) { + if (connInfo.baseUrl.startsWith('https') && allowUnauthorized) { const requestAgent = new HttpsAgent({ rejectUnauthorized: false }); requestInit = { ...requestInit, agent: requestAgent }; - allowUnauthorized = true; } // This replaces the WebSocket constructor in jupyter lab services with our own implementation diff --git a/src/client/datascience/jupyter/jupyterSessionManagerFactory.ts b/src/client/datascience/jupyter/jupyterSessionManagerFactory.ts index bef26532ba1c..59210d84df48 100644 --- a/src/client/datascience/jupyter/jupyterSessionManagerFactory.ts +++ b/src/client/datascience/jupyter/jupyterSessionManagerFactory.ts @@ -34,7 +34,8 @@ export class JupyterSessionManagerFactory implements IJupyterSessionManagerFacto this.config, failOnPassword, this.kernelSelector, - this.jupyterOutput + this.jupyterOutput, + this.config ); await result.initialize(connInfo); return result; diff --git a/src/client/datascience/jupyter/jupyterUtils.ts b/src/client/datascience/jupyter/jupyterUtils.ts index 0eca35d6dddc..ffd17e2c827e 100644 --- a/src/client/datascience/jupyter/jupyterUtils.ts +++ b/src/client/datascience/jupyter/jupyterUtils.ts @@ -7,7 +7,6 @@ import * as path from 'path'; import { Uri } from 'vscode'; import { IWorkspaceService } from '../../common/application/types'; -import { IDataScienceSettings } from '../../common/types'; import { noop } from '../../common/utils/misc'; import { SystemVariables } from '../../common/variables/systemVariables'; import { getJupyterConnectionDisplayName } from '../jupyter/jupyterConnection'; @@ -27,7 +26,7 @@ export function expandWorkingDir( return path.dirname(launchingFile); } -export function createRemoteConnectionInfo(uri: string, settings: IDataScienceSettings): IJupyterConnection { +export function createRemoteConnectionInfo(uri: string): IJupyterConnection { let url: URL; try { url = new URL(uri); @@ -35,16 +34,12 @@ export function createRemoteConnectionInfo(uri: string, settings: IDataScienceSe // This should already have been parsed when set, so just throw if it's not right here throw err; } - const allowUnauthorized = settings.allowUnauthorizedRemoteConnection - ? settings.allowUnauthorizedRemoteConnection - : false; const baseUrl = `${url.protocol}//${url.host}${url.pathname}`; const token = `${url.searchParams.get('token')}`; return { type: 'jupyter', - allowUnauthorized, baseUrl, token, hostName: url.hostname, diff --git a/src/client/datascience/types.ts b/src/client/datascience/types.ts index 71ce472ffd12..0b55aa38d208 100644 --- a/src/client/datascience/types.ts +++ b/src/client/datascience/types.ts @@ -72,7 +72,6 @@ export interface IJupyterConnection extends Disposable { readonly token: string; readonly hostName: string; localProcExitCode: number | undefined; - allowUnauthorized?: boolean; } export type INotebookProviderConnection = IRawConnection | IJupyterConnection; @@ -313,10 +312,7 @@ export interface IJupyterPasswordConnectInfo { export const IJupyterPasswordConnect = Symbol('IJupyterPasswordConnect'); export interface IJupyterPasswordConnect { - getPasswordConnectionInfo( - url: string, - allowUnauthorized: boolean - ): Promise; + getPasswordConnectionInfo(url: string): Promise; } export const IJupyterSession = Symbol('IJupyterSession'); diff --git a/src/test/datascience/jupyter/jupyterConnection.unit.test.ts b/src/test/datascience/jupyter/jupyterConnection.unit.test.ts index 8a8674c849b9..26dcde50e2b5 100644 --- a/src/test/datascience/jupyter/jupyterConnection.unit.test.ts +++ b/src/test/datascience/jupyter/jupyterConnection.unit.test.ts @@ -106,7 +106,6 @@ suite('Data Science - JupyterConnection', () => { const connection = await waiter.waitForConnection(); assert.equal(connection.localLaunch, true); - assert.equal(connection.allowUnauthorized, undefined); assert.equal(connection.localProcExitCode, undefined); assert.equal(connection.baseUrl, expectedServerInfo.url); assert.equal(connection.hostName, expectedServerInfo.hostname); diff --git a/src/test/datascience/jupyterPasswordConnect.unit.test.ts b/src/test/datascience/jupyterPasswordConnect.unit.test.ts index b2d669563c20..f580b5925d9e 100644 --- a/src/test/datascience/jupyterPasswordConnect.unit.test.ts +++ b/src/test/datascience/jupyterPasswordConnect.unit.test.ts @@ -9,6 +9,7 @@ import * as typemoq from 'typemoq'; import { instance, mock } from 'ts-mockito'; import { IApplicationShell } from '../../client/common/application/types'; import { AsyncDisposableRegistry } from '../../client/common/asyncDisposableRegistry'; +import { ConfigurationService } from '../../client/common/configuration/service'; import { MultiStepInputFactory } from '../../client/common/utils/multiStepInput'; import { JupyterPasswordConnect } from '../../client/datascience/jupyter/jupyterPasswordConnect'; @@ -26,11 +27,13 @@ suite('JupyterPasswordConnect', () => { appShell.setup((a) => a.showInputBox(typemoq.It.isAny())).returns(() => Promise.resolve('Python')); const multiStepFactory = new MultiStepInputFactory(appShell.object); const mockDisposableRegistry = mock(AsyncDisposableRegistry); + const mockConfigSettings = mock(ConfigurationService); jupyterPasswordConnect = new JupyterPasswordConnect( appShell.object, multiStepFactory, - instance(mockDisposableRegistry) + instance(mockDisposableRegistry), + instance(mockConfigSettings) ); }); @@ -123,7 +126,6 @@ suite('JupyterPasswordConnect', () => { const result = await jupyterPasswordConnect.getPasswordConnectionInfo( //tslint:disable-next-line:no-http-string 'http://TESTNAME:8888/', - false, fetchMock.object ); assert(result, 'Failed to get password'); @@ -174,7 +176,6 @@ suite('JupyterPasswordConnect', () => { //tslint:disable-next-line:no-http-string const result = await jupyterPasswordConnect.getPasswordConnectionInfo( 'https://TESTNAME:8888/', - true, fetchMock.object ); assert(result, 'Failed to get password'); @@ -197,7 +198,6 @@ suite('JupyterPasswordConnect', () => { const result = await jupyterPasswordConnect.getPasswordConnectionInfo( //tslint:disable-next-line:no-http-string 'http://TESTNAME:8888/', - false, fetchMock.object ); assert(!result); From 3e8402c7e6dc31d76f9cb7dcce4c2c399f397361 Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Wed, 24 Jun 2020 16:50:10 -0700 Subject: [PATCH 07/14] Skip waiting on shutdown --- src/client/datascience/jupyter/jupyterPasswordConnect.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/datascience/jupyter/jupyterPasswordConnect.ts b/src/client/datascience/jupyter/jupyterPasswordConnect.ts index deda5376c53c..a45c0f1d44a3 100644 --- a/src/client/datascience/jupyter/jupyterPasswordConnect.ts +++ b/src/client/datascience/jupyter/jupyterPasswordConnect.ts @@ -162,14 +162,14 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { // the user closes VS code this.asyncDisposableRegistry.push({ dispose: async () => { - await this.makeRequest(`${baseUrl}/hub/api/users/${username}/tokens/${body.id}`, { + this.makeRequest(`${baseUrl}/hub/api/users/${username}/tokens/${body.id}`, { method: 'DELETE', headers: { Connection: 'keep-alive', Cookie: cookieString, Referer: `${baseUrl}/hub/login` } - }); + }).ignoreErrors(); // Don't wait for this during shutdown. Just make the request } }); From 7cc2ce43b2ee5db204365fb03e93a5fd16afc11d Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Wed, 24 Jun 2020 17:23:31 -0700 Subject: [PATCH 08/14] Get old unit tests passing again --- .../jupyterPasswordConnect.unit.test.ts | 45 ++++++++++++++----- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/src/test/datascience/jupyterPasswordConnect.unit.test.ts b/src/test/datascience/jupyterPasswordConnect.unit.test.ts index f580b5925d9e..deac320ab99c 100644 --- a/src/test/datascience/jupyterPasswordConnect.unit.test.ts +++ b/src/test/datascience/jupyterPasswordConnect.unit.test.ts @@ -6,7 +6,7 @@ import * as assert from 'assert'; import * as nodeFetch from 'node-fetch'; import * as typemoq from 'typemoq'; -import { instance, mock } from 'ts-mockito'; +import { anything, instance, mock, when } from 'ts-mockito'; import { IApplicationShell } from '../../client/common/application/types'; import { AsyncDisposableRegistry } from '../../client/common/asyncDisposableRegistry'; import { ConfigurationService } from '../../client/common/configuration/service'; @@ -17,6 +17,7 @@ import { JupyterPasswordConnect } from '../../client/datascience/jupyter/jupyter suite('JupyterPasswordConnect', () => { let jupyterPasswordConnect: JupyterPasswordConnect; let appShell: typemoq.IMock; + let configService: ConfigurationService; const xsrfValue: string = '12341234'; const sessionName: string = 'sessionName'; @@ -27,17 +28,28 @@ suite('JupyterPasswordConnect', () => { appShell.setup((a) => a.showInputBox(typemoq.It.isAny())).returns(() => Promise.resolve('Python')); const multiStepFactory = new MultiStepInputFactory(appShell.object); const mockDisposableRegistry = mock(AsyncDisposableRegistry); - const mockConfigSettings = mock(ConfigurationService); + configService = mock(ConfigurationService); jupyterPasswordConnect = new JupyterPasswordConnect( appShell.object, multiStepFactory, instance(mockDisposableRegistry), - instance(mockConfigSettings) + instance(configService) ); }); function createMockSetup(secure: boolean, ok: boolean) { + const dsSettings = { + allowUnauthorizedRemoteConnection: secure + // tslint:disable-next-line: no-any + } as any; + when(configService.getSettings(anything())).thenReturn({ datascience: dsSettings } as any); + when(configService.updateSetting('dataScience.jupyterServerURI', anything(), anything(), anything())).thenCall( + (_a1, _a2, _a3, _a4) => { + return Promise.resolve(); + } + ); + // Set up our fake node fetch const fetchMock: typemoq.IMock = typemoq.Mock.ofInstance(nodeFetch.default); @@ -47,7 +59,11 @@ suite('JupyterPasswordConnect', () => { // Mock our first call to get xsrf cookie const mockXsrfResponse = typemoq.Mock.ofType(nodeFetch.Response); const mockXsrfHeaders = typemoq.Mock.ofType(nodeFetch.Headers); - mockXsrfHeaders.setup((mh) => mh.get('set-cookie')).returns(() => `_xsrf=${xsrfValue}`); + mockXsrfHeaders + .setup((mh) => mh.raw()) + .returns(() => { + return { 'set-cookie': [`_xsrf=${xsrfValue}`] }; + }); mockXsrfResponse.setup((mr) => mr.ok).returns(() => ok); mockXsrfResponse.setup((mr) => mr.status).returns(() => 302); mockXsrfResponse.setup((mr) => mr.headers).returns(() => mockXsrfHeaders.object); @@ -101,7 +117,13 @@ suite('JupyterPasswordConnect', () => { // Mock our second call to get session cookie const mockSessionResponse = typemoq.Mock.ofType(nodeFetch.Response); const mockSessionHeaders = typemoq.Mock.ofType(nodeFetch.Headers); - mockSessionHeaders.setup((mh) => mh.get('set-cookie')).returns(() => `${sessionName}=${sessionValue}`); + mockSessionHeaders + .setup((mh) => mh.raw()) + .returns(() => { + return { + 'set-cookie': [`${sessionName}=${sessionValue}`] + }; + }); mockSessionResponse.setup((mr) => mr.status).returns(() => 302); mockSessionResponse.setup((mr) => mr.headers).returns(() => mockSessionHeaders.object); @@ -131,7 +153,7 @@ suite('JupyterPasswordConnect', () => { assert(result, 'Failed to get password'); if (result) { // tslint:disable-next-line: no-cookies - assert.ok((result.requestHeaders as any).cookie, 'No cookie'); + assert.ok((result.requestHeaders as any).Cookie, 'No cookie'); } // Verfiy calls @@ -149,9 +171,12 @@ suite('JupyterPasswordConnect', () => { const mockSessionResponse = typemoq.Mock.ofType(nodeFetch.Response); const mockSessionHeaders = typemoq.Mock.ofType(nodeFetch.Headers); mockSessionHeaders - .setup((mh) => mh.get('set-cookie')) - .returns(() => `${sessionName}=${sessionValue}`) - .verifiable(typemoq.Times.once()); + .setup((mh) => mh.raw()) + .returns(() => { + return { + 'set-cookie': [`${sessionName}=${sessionValue}`] + }; + }); mockSessionResponse.setup((mr) => mr.status).returns(() => 302); mockSessionResponse.setup((mr) => mr.headers).returns(() => mockSessionHeaders.object); @@ -181,7 +206,7 @@ suite('JupyterPasswordConnect', () => { assert(result, 'Failed to get password'); if (result) { // tslint:disable-next-line: no-cookies - assert.ok((result.requestHeaders as any).cookie, 'No cookie'); + assert.ok((result.requestHeaders as any).Cookie, 'No cookie'); } // Verfiy calls From 89264679ed762a781cd4435f02914a4c21665fbb Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Wed, 24 Jun 2020 17:34:43 -0700 Subject: [PATCH 09/14] Fix xsrf token to remove extra cookie crap --- src/client/datascience/jupyter/jupyterPasswordConnect.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/datascience/jupyter/jupyterPasswordConnect.ts b/src/client/datascience/jupyter/jupyterPasswordConnect.ts index a45c0f1d44a3..805969b7262c 100644 --- a/src/client/datascience/jupyter/jupyterPasswordConnect.ts +++ b/src/client/datascience/jupyter/jupyterPasswordConnect.ts @@ -339,7 +339,7 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { if (response.ok) { const cookies = this.getCookies(response); if (cookies.has('_xsrf')) { - xsrfCookie = cookies.get('_xsrf'); + xsrfCookie = cookies.get('_xsrf')?.split(';')[0]; } } From 502992e5a12b6a040144dc9e3188a8f6fe2fb13a Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Thu, 25 Jun 2020 11:39:36 -0700 Subject: [PATCH 10/14] Test plan changes and new unit test --- .github/test_plan.md | 31 ++++++ news/1 Enhancements/9679.md | 2 + .../jupyterPasswordConnect.unit.test.ts | 94 ++++++++++++++++--- 3 files changed, 112 insertions(+), 15 deletions(-) create mode 100644 news/1 Enhancements/9679.md diff --git a/.github/test_plan.md b/.github/test_plan.md index 0147854f8a9f..013274920c65 100644 --- a/.github/test_plan.md +++ b/.github/test_plan.md @@ -575,6 +575,37 @@ def test_failure(): 1. Open the notebook editor on the host 1. Run a cell on the host 1. Verify the editor opens on the guest and the cell is run there too +- [ ] Jupyter Hub support + 1. Install Docker Desktop onto a machine + 1. Create a folder with a file 'Dockerfile' in it. + 1. Mark the file to look like so: + + ``` + ARG BASE_CONTAINER=jupyterhub/jupyterhub + FROM $BASE_CONTAINER + + USER root + + USER $NB_UID + ``` + + 1. From a command prompt (in the same folder as the Dockerfile), run ```docker build -t jupyterhubcontainer:1.0 .``` + 1. Run ```docker container create --name jupyterhub jupyterhubcontainer:1.0 jupyterhub``` + 1. From the docker desktop app, start the jupyterhub container. + 1. From the docker desktop app, run the CLI + 1. From the new command prompt, run ```adduser testuser``` + 1. Follow the series of prompts to add a password for this user + 1. Open VS code + 1. Open a folder with a python file in it. + 1. Run the ```Python: Specify local or remote Jupyter server for connections``` command. + 1. Pick 'Existing' + 1. Enter ```http://localhost:8000``` (assuming the jupyter hub container was successful in launching) + 1. Reload VS code and reopen this folder. + 1. Run a cell in a python file. + [ ] Verify results + 1. Verify you are asked first for a user name and then a password. + 1. Verify a cell runs once you enter the user name and password + 1. Verify that the python that is running in the interactive window is from the docker container (if on windows it should show a linux path) #### P2 Test Scenarios diff --git a/news/1 Enhancements/9679.md b/news/1 Enhancements/9679.md new file mode 100644 index 000000000000..504ad7d7cc76 --- /dev/null +++ b/news/1 Enhancements/9679.md @@ -0,0 +1,2 @@ +Support connecting to Jupyter hub servers. Use either the base url of the server (i.e. 'https://111.11.11.11:8000') or your user folder (i.e. 'https://111.11.11.11:8000/user/theuser). +Works with password authentication. \ No newline at end of file diff --git a/src/test/datascience/jupyterPasswordConnect.unit.test.ts b/src/test/datascience/jupyterPasswordConnect.unit.test.ts index deac320ab99c..bfcccdea7cad 100644 --- a/src/test/datascience/jupyterPasswordConnect.unit.test.ts +++ b/src/test/datascience/jupyterPasswordConnect.unit.test.ts @@ -7,16 +7,19 @@ import * as nodeFetch from 'node-fetch'; import * as typemoq from 'typemoq'; import { anything, instance, mock, when } from 'ts-mockito'; +import { ApplicationShell } from '../../client/common/application/applicationShell'; import { IApplicationShell } from '../../client/common/application/types'; import { AsyncDisposableRegistry } from '../../client/common/asyncDisposableRegistry'; import { ConfigurationService } from '../../client/common/configuration/service'; import { MultiStepInputFactory } from '../../client/common/utils/multiStepInput'; import { JupyterPasswordConnect } from '../../client/datascience/jupyter/jupyterPasswordConnect'; +import { MockInputBox } from './mockInputBox'; +import { MockQuickPick } from './mockQuickPick'; -// tslint:disable:no-any max-func-body-length +// tslint:disable:no-any max-func-body-length no-http-string suite('JupyterPasswordConnect', () => { let jupyterPasswordConnect: JupyterPasswordConnect; - let appShell: typemoq.IMock; + let appShell: ApplicationShell; let configService: ConfigurationService; const xsrfValue: string = '12341234'; @@ -24,14 +27,14 @@ suite('JupyterPasswordConnect', () => { const sessionValue: string = 'sessionValue'; setup(() => { - appShell = typemoq.Mock.ofType(); - appShell.setup((a) => a.showInputBox(typemoq.It.isAny())).returns(() => Promise.resolve('Python')); - const multiStepFactory = new MultiStepInputFactory(appShell.object); + appShell = mock(ApplicationShell); + when(appShell.showInputBox(anything(), anything())).thenReturn(Promise.resolve('Python')); + const multiStepFactory = new MultiStepInputFactory(instance(appShell)); const mockDisposableRegistry = mock(AsyncDisposableRegistry); configService = mock(ConfigurationService); jupyterPasswordConnect = new JupyterPasswordConnect( - appShell.object, + instance(appShell), multiStepFactory, instance(mockDisposableRegistry), instance(configService) @@ -52,8 +55,6 @@ suite('JupyterPasswordConnect', () => { // Set up our fake node fetch const fetchMock: typemoq.IMock = typemoq.Mock.ofInstance(nodeFetch.default); - - //tslint:disable-next-line:no-http-string const rootUrl = secure ? 'https://TESTNAME:8888/' : 'http://TESTNAME:8888/'; // Mock our first call to get xsrf cookie @@ -85,7 +86,6 @@ suite('JupyterPasswordConnect', () => { .returns(() => Promise.resolve(mockXsrfResponse.object)); fetchMock .setup((fm) => - //tslint:disable-next-line:no-http-string fm( `${rootUrl}tree?`, typemoq.It.isObjectWith({ @@ -97,7 +97,6 @@ suite('JupyterPasswordConnect', () => { .returns(() => Promise.resolve(mockXsrfResponse.object)); fetchMock .setup((fm) => - //tslint:disable-next-line:no-http-string fm( `${rootUrl}hub/api`, typemoq.It.isObjectWith({ @@ -131,7 +130,6 @@ suite('JupyterPasswordConnect', () => { fetchMock .setup((fm) => fm( - //tslint:disable-next-line:no-http-string 'http://TESTNAME:8888/login?', typemoq.It.isObjectWith({ method: 'post', @@ -146,7 +144,6 @@ suite('JupyterPasswordConnect', () => { .returns(() => Promise.resolve(mockSessionResponse.object)); const result = await jupyterPasswordConnect.getPasswordConnectionInfo( - //tslint:disable-next-line:no-http-string 'http://TESTNAME:8888/', fetchMock.object ); @@ -181,7 +178,6 @@ suite('JupyterPasswordConnect', () => { mockSessionResponse.setup((mr) => mr.headers).returns(() => mockSessionHeaders.object); // typemoq doesn't love this comparison, so generalize it a bit - //tslint:disable-next-line:no-http-string fetchMock .setup((fm) => fm( @@ -198,7 +194,6 @@ suite('JupyterPasswordConnect', () => { ) .returns(() => Promise.resolve(mockSessionResponse.object)); - //tslint:disable-next-line:no-http-string const result = await jupyterPasswordConnect.getPasswordConnectionInfo( 'https://TESTNAME:8888/', fetchMock.object @@ -221,7 +216,6 @@ suite('JupyterPasswordConnect', () => { const { fetchMock, mockXsrfHeaders, mockXsrfResponse } = createMockSetup(false, false); const result = await jupyterPasswordConnect.getPasswordConnectionInfo( - //tslint:disable-next-line:no-http-string 'http://TESTNAME:8888/', fetchMock.object ); @@ -232,4 +226,74 @@ suite('JupyterPasswordConnect', () => { mockXsrfResponse.verifyAll(); fetchMock.verifyAll(); }); + + function createJupyterHubSetup() { + const dsSettings = { + allowUnauthorizedRemoteConnection: false + // tslint:disable-next-line: no-any + } as any; + when(configService.getSettings(anything())).thenReturn({ datascience: dsSettings } as any); + when(configService.updateSetting('dataScience.jupyterServerURI', anything(), anything(), anything())).thenCall( + (_a1, _a2, _a3, _a4) => { + return Promise.resolve(); + } + ); + + const quickPick = new MockQuickPick(''); + const input = new MockInputBox('test'); + when(appShell.createQuickPick()).thenReturn(quickPick!); + when(appShell.createInputBox()).thenReturn(input); + + const hubActiveResponse = mock(nodeFetch.Response); + when(hubActiveResponse.ok).thenReturn(true); + when(hubActiveResponse.status).thenReturn(200); + const invalidResponse = mock(nodeFetch.Response); + when(invalidResponse.ok).thenReturn(false); + when(invalidResponse.status).thenReturn(404); + const loginResponse = mock(nodeFetch.Response); + const loginHeaders = mock(nodeFetch.Headers); + when(loginHeaders.raw()).thenReturn({ 'set-cookie': ['super-cookie-login=foobar'] }); + when(loginResponse.ok).thenReturn(true); + when(loginResponse.status).thenReturn(302); + when(loginResponse.headers).thenReturn(instance(loginHeaders)); + const tokenResponse = mock(nodeFetch.Response); + when(tokenResponse.ok).thenReturn(true); + when(tokenResponse.status).thenReturn(200); + when(tokenResponse.json()).thenResolve({ + token: 'foobar', + id: '1' + }); + + instance(hubActiveResponse as any).then = undefined; + instance(invalidResponse as any).then = undefined; + instance(loginResponse as any).then = undefined; + instance(tokenResponse as any).then = undefined; + + return async (url: nodeFetch.RequestInfo, init?: nodeFetch.RequestInit) => { + const urlString = url.toString().toLowerCase(); + if (urlString === 'http://testname:8888/hub/api') { + return instance(hubActiveResponse); + } else if (urlString === 'http://testname:8888/hub/login?next=') { + return instance(loginResponse); + } else if ( + urlString === 'http://testname:8888/hub/api/users/test/tokens' && + init && + init.method === 'POST' && + (init.headers as any).Referer === 'http://testname:8888/hub/login' && + (init.headers as any).Cookie === ';super-cookie-login=foobar' + ) { + return instance(tokenResponse); + } + return instance(invalidResponse); + }; + } + test('getPasswordConnectionInfo jupyter hub', async () => { + const fetchMock = createJupyterHubSetup(); + + const result = await jupyterPasswordConnect.getPasswordConnectionInfo('http://TESTNAME:8888/', fetchMock); + assert.ok(result, 'No hub connection info'); + assert.equal(result?.remappedBaseUrl, 'http://testname:8888/user/test', 'Url not remapped'); + assert.equal(result?.remappedToken, 'foobar', 'Token should be returned in URL'); + assert.ok(result?.requestHeaders, 'No request headers returned for jupyter hub'); + }); }); From 252efca013c90edaed8071393d828535b2304b6a Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Thu, 25 Jun 2020 11:44:15 -0700 Subject: [PATCH 11/14] Fix linter and input box result --- src/test/datascience/jupyterPasswordConnect.unit.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/test/datascience/jupyterPasswordConnect.unit.test.ts b/src/test/datascience/jupyterPasswordConnect.unit.test.ts index bfcccdea7cad..1c90275cc011 100644 --- a/src/test/datascience/jupyterPasswordConnect.unit.test.ts +++ b/src/test/datascience/jupyterPasswordConnect.unit.test.ts @@ -8,7 +8,6 @@ import * as typemoq from 'typemoq'; import { anything, instance, mock, when } from 'ts-mockito'; import { ApplicationShell } from '../../client/common/application/applicationShell'; -import { IApplicationShell } from '../../client/common/application/types'; import { AsyncDisposableRegistry } from '../../client/common/asyncDisposableRegistry'; import { ConfigurationService } from '../../client/common/configuration/service'; import { MultiStepInputFactory } from '../../client/common/utils/multiStepInput'; @@ -28,7 +27,7 @@ suite('JupyterPasswordConnect', () => { setup(() => { appShell = mock(ApplicationShell); - when(appShell.showInputBox(anything(), anything())).thenReturn(Promise.resolve('Python')); + when(appShell.showInputBox(anything())).thenReturn(Promise.resolve('Python')); const multiStepFactory = new MultiStepInputFactory(instance(appShell)); const mockDisposableRegistry = mock(AsyncDisposableRegistry); configService = mock(ConfigurationService); From 5733fec262bfae69fa3ef62da9c477d36b015b28 Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Thu, 25 Jun 2020 12:17:25 -0700 Subject: [PATCH 12/14] Fix sonar warning --- .../jupyter/jupyterPasswordConnect.ts | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/client/datascience/jupyter/jupyterPasswordConnect.ts b/src/client/datascience/jupyter/jupyterPasswordConnect.ts index 805969b7262c..0f1850218be8 100644 --- a/src/client/datascience/jupyter/jupyterPasswordConnect.ts +++ b/src/client/datascience/jupyter/jupyterPasswordConnect.ts @@ -100,13 +100,7 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { password: string ): Promise { // We're using jupyter hub. Get the base url - let url: URL; - try { - url = new URL(uri); - } catch (err) { - // This should already have been parsed when set, so just throw if it's not right here - throw err; - } + const url = new URL(uri); const baseUrl = `${url.protocol}//${url.host}`; const postParams = new URLSearchParams(); @@ -189,13 +183,7 @@ export class JupyterPasswordConnect implements IJupyterPasswordConnect { password: string ): Promise { // We're using jupyter hub. Get the base url - let url: URL; - try { - url = new URL(uri); - } catch (err) { - // This should already have been parsed when set, so just throw if it's not right here - throw err; - } + const url = new URL(uri); const baseUrl = `${url.protocol}//${url.host}`; // Use these in a post request to get the token to use const response = await this.makeRequest( From 695618e6b3c96555833ce394000e64a35acaa101 Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Thu, 25 Jun 2020 12:38:12 -0700 Subject: [PATCH 13/14] Review feedback --- package.nls.json | 2 +- src/client/common/utils/localize.ts | 2 +- src/client/datascience/jupyter/jupyterSessionManager.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.nls.json b/package.nls.json index 133f74e65090..8afd098b54a0 100644 --- a/package.nls.json +++ b/package.nls.json @@ -240,7 +240,7 @@ "DataScience.jupyterInstall": "Install", "DataScience.jupyterSelectURIPrompt": "Enter the URI of the running Jupyter server", "DataScience.jupyterSelectURIInvalidURI": "Invalid URI specified", - "DataScience.jupyterSelectUserAndPasswordTitle": "Enter your user name and password to connect to jupyter hub", + "DataScience.jupyterSelectUserAndPasswordTitle": "Enter your user name and password to connect to Jupyter Hub", "DataScience.jupyterSelectUserPrompt": "Enter your user name", "DataScience.jupyterSelectPasswordPrompt": "Enter your password", "DataScience.jupyterNotebookFailure": "Jupyter notebook failed to launch. \r\n{0}", diff --git a/src/client/common/utils/localize.ts b/src/client/common/utils/localize.ts index d7533a721ebb..88508658f3de 100644 --- a/src/client/common/utils/localize.ts +++ b/src/client/common/utils/localize.ts @@ -522,7 +522,7 @@ export namespace DataScience { ); export const jupyterSelectUserAndPasswordTitle = localize( 'DataScience.jupyterSelectUserAndPasswordTitle', - 'Enter your user name and password to connect to jupyter hub' + 'Enter your user name and password to connect to Jupyter Hub' ); export const jupyterSelectUserPrompt = localize('DataScience.jupyterSelectUserPrompt', 'Enter your user name'); export const jupyterSelectPasswordPrompt = localize( diff --git a/src/client/datascience/jupyter/jupyterSessionManager.ts b/src/client/datascience/jupyter/jupyterSessionManager.ts index 6a0816e70c31..555dace77dcc 100644 --- a/src/client/datascience/jupyter/jupyterSessionManager.ts +++ b/src/client/datascience/jupyter/jupyterSessionManager.ts @@ -240,7 +240,7 @@ export class JupyterSessionManager implements IJupyterSessionManager { } serverSettings = { ...serverSettings, token: '' }; const pwSettings = await this.jupyterPasswordConnect.getPasswordConnectionInfo(connInfo.baseUrl); - if (pwSettings && pwSettings.requestHeaders) { + if (pwSettings && pwSettings.requestHeaders && (pwSettings.requestHeaders as any).Cookie) { requestInit = { ...requestInit, headers: pwSettings.requestHeaders }; cookieString = (pwSettings.requestHeaders as any).Cookie; From 30f9f30836725e86086c610a367083e2058a993c Mon Sep 17 00:00:00 2001 From: Rich Chiodo Date: Thu, 25 Jun 2020 12:45:13 -0700 Subject: [PATCH 14/14] Actually cookie not sent back for hub --- src/client/datascience/jupyter/jupyterSessionManager.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/datascience/jupyter/jupyterSessionManager.ts b/src/client/datascience/jupyter/jupyterSessionManager.ts index 555dace77dcc..189d83ab3249 100644 --- a/src/client/datascience/jupyter/jupyterSessionManager.ts +++ b/src/client/datascience/jupyter/jupyterSessionManager.ts @@ -240,9 +240,9 @@ export class JupyterSessionManager implements IJupyterSessionManager { } serverSettings = { ...serverSettings, token: '' }; const pwSettings = await this.jupyterPasswordConnect.getPasswordConnectionInfo(connInfo.baseUrl); - if (pwSettings && pwSettings.requestHeaders && (pwSettings.requestHeaders as any).Cookie) { + if (pwSettings && pwSettings.requestHeaders) { requestInit = { ...requestInit, headers: pwSettings.requestHeaders }; - cookieString = (pwSettings.requestHeaders as any).Cookie; + cookieString = (pwSettings.requestHeaders as any).Cookie || ''; // Password may have overwritten the base url and token as well if (pwSettings.remappedBaseUrl) {