diff --git a/package.json b/package.json index e2a93ec80a2c..116927235ec5 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "theme": "dark" }, "engines": { - "vscode": "^1.18.0" + "vscode": "^1.23.0" }, "recommendations": [ "donjayamanne.jupyter" diff --git a/src/client/extension.ts b/src/client/extension.ts index b14c115bb176..723e471c0761 100644 --- a/src/client/extension.ts +++ b/src/client/extension.ts @@ -1,195 +1,197 @@ -'use strict'; -// This line should always be right on top. -// tslint:disable-next-line:no-any -if ((Reflect as any).metadata === undefined) { - // tslint:disable-next-line:no-require-imports no-var-requires - require('reflect-metadata'); -} -import { Container } from 'inversify'; -import { - debug, Disposable, ExtensionContext, - extensions, IndentAction, languages, Memento, - OutputChannel, window -} from 'vscode'; -import { AnalysisExtensionActivator } from './activation/analysis'; -import { ClassicExtensionActivator } from './activation/classic'; -import { IExtensionActivator } from './activation/types'; -import { PythonSettings } from './common/configSettings'; -import { isPythonAnalysisEngineTest, PYTHON, PYTHON_LANGUAGE, STANDARD_OUTPUT_CHANNEL } from './common/constants'; -import { FeatureDeprecationManager } from './common/featureDeprecationManager'; -import { createDeferred } from './common/helpers'; -import { PythonInstaller } from './common/installer/pythonInstallation'; -import { registerTypes as installerRegisterTypes } from './common/installer/serviceRegistry'; -import { registerTypes as platformRegisterTypes } from './common/platform/serviceRegistry'; -import { registerTypes as processRegisterTypes } from './common/process/serviceRegistry'; -import { registerTypes as commonRegisterTypes } from './common/serviceRegistry'; -import { StopWatch } from './common/stopWatch'; -import { GLOBAL_MEMENTO, IConfigurationService, IDisposableRegistry, ILogger, IMemento, IOutputChannel, IPersistentStateFactory, WORKSPACE_MEMENTO } from './common/types'; -import { registerTypes as variableRegisterTypes } from './common/variables/serviceRegistry'; -import { AttachRequestArguments, LaunchRequestArguments } from './debugger/Common/Contracts'; -import { BaseConfigurationProvider } from './debugger/configProviders/baseProvider'; -import { registerTypes as debugConfigurationRegisterTypes } from './debugger/configProviders/serviceRegistry'; -import { IDebugConfigurationProvider } from './debugger/types'; -import { registerTypes as formattersRegisterTypes } from './formatters/serviceRegistry'; -import { IInterpreterSelector } from './interpreter/configuration/types'; -import { ICondaService, IInterpreterService } from './interpreter/contracts'; -import { registerTypes as interpretersRegisterTypes } from './interpreter/serviceRegistry'; -import { ServiceContainer } from './ioc/container'; -import { ServiceManager } from './ioc/serviceManager'; -import { IServiceContainer } from './ioc/types'; -import { LinterCommands } from './linters/linterCommands'; -import { registerTypes as lintersRegisterTypes } from './linters/serviceRegistry'; -import { ILintingEngine } from './linters/types'; -import { PythonFormattingEditProvider } from './providers/formatProvider'; -import { LinterProvider } from './providers/linterProvider'; -import { ReplProvider } from './providers/replProvider'; -import { TerminalProvider } from './providers/terminalProvider'; -import { activateUpdateSparkLibraryProvider } from './providers/updateSparkLibraryProvider'; -import * as sortImports from './sortImports'; -import { sendTelemetryEvent } from './telemetry'; -import { EDITOR_LOAD } from './telemetry/constants'; -import { registerTypes as commonRegisterTerminalTypes } from './terminals/serviceRegistry'; -import { ICodeExecutionManager } from './terminals/types'; -import { BlockFormatProviders } from './typeFormatters/blockFormatProvider'; -import { OnEnterFormatter } from './typeFormatters/onEnterFormatter'; -import { TEST_OUTPUT_CHANNEL } from './unittests/common/constants'; -import { registerTypes as unitTestsRegisterTypes } from './unittests/serviceRegistry'; -import { WorkspaceSymbols } from './workspaceSymbols/main'; - -const activationDeferred = createDeferred(); -export const activated = activationDeferred.promise; - -// tslint:disable-next-line:max-func-body-length -export async function activate(context: ExtensionContext) { - const cont = new Container(); - const serviceManager = new ServiceManager(cont); - const serviceContainer = new ServiceContainer(cont); - registerServices(context, serviceManager, serviceContainer); - - const interpreterManager = serviceContainer.get(IInterpreterService); - // This must be completed before we can continue as language server needs the interpreter path. - interpreterManager.initialize(); - await interpreterManager.autoSetInterpreter(); - - const configuration = serviceManager.get(IConfigurationService); - const pythonSettings = configuration.getSettings(); - - const activator: IExtensionActivator = isPythonAnalysisEngineTest() || !pythonSettings.jediEnabled - ? new AnalysisExtensionActivator(serviceManager, pythonSettings) - : new ClassicExtensionActivator(serviceManager, pythonSettings, PYTHON); - - await activator.activate(context); - - const standardOutputChannel = serviceManager.get(IOutputChannel, STANDARD_OUTPUT_CHANNEL); - sortImports.activate(context, standardOutputChannel, serviceManager); - - serviceManager.get(ICodeExecutionManager).registerCommands(); - // tslint:disable-next-line:no-floating-promises - sendStartupTelemetry(activated, serviceContainer); - - const pythonInstaller = new PythonInstaller(serviceContainer); - pythonInstaller.checkPythonInstallation(PythonSettings.getInstance()) - .catch(ex => console.error('Python Extension: pythonInstaller.checkPythonInstallation', ex)); - - interpreterManager.refresh() - .catch(ex => console.error('Python Extension: interpreterManager.refresh', ex)); - - const jupyterExtension = extensions.getExtension('donjayamanne.jupyter'); - const lintingEngine = serviceManager.get(ILintingEngine); - lintingEngine.linkJupiterExtension(jupyterExtension).ignoreErrors(); - - context.subscriptions.push(new LinterCommands(serviceManager)); - const linterProvider = new LinterProvider(context, serviceManager); - context.subscriptions.push(linterProvider); - - // Enable indentAction - // tslint:disable-next-line:no-non-null-assertion - languages.setLanguageConfiguration(PYTHON_LANGUAGE, { - onEnterRules: [ - { - beforeText: /^\s*(?:def|class|for|if|elif|else|while|try|with|finally|except)\b.*:\s*\S+/, - action: { indentAction: IndentAction.None } - }, - { - beforeText: /^\s*(?:def|class|for|if|elif|else|while|try|with|finally|except|async)\b.*:\s*/, - action: { indentAction: IndentAction.Indent } - }, - { - beforeText: /^\s*#.*/, - afterText: /.+$/, - action: { indentAction: IndentAction.None, appendText: '# ' } - }, - { - beforeText: /^\s+(continue|break|return)\b.*/, - afterText: /\s+$/, - action: { indentAction: IndentAction.Outdent } - } - ] - }); - - if (pythonSettings && pythonSettings.formatting && pythonSettings.formatting.provider !== 'none') { - const formatProvider = new PythonFormattingEditProvider(context, serviceContainer); - context.subscriptions.push(languages.registerDocumentFormattingEditProvider(PYTHON, formatProvider)); - context.subscriptions.push(languages.registerDocumentRangeFormattingEditProvider(PYTHON, formatProvider)); - } - - context.subscriptions.push(languages.registerOnTypeFormattingEditProvider(PYTHON, new BlockFormatProviders(), ':')); - context.subscriptions.push(languages.registerOnTypeFormattingEditProvider(PYTHON, new OnEnterFormatter(), '\n')); - - const persistentStateFactory = serviceManager.get(IPersistentStateFactory); - const deprecationMgr = new FeatureDeprecationManager(persistentStateFactory, !!jupyterExtension); - deprecationMgr.initialize(); - context.subscriptions.push(new FeatureDeprecationManager(persistentStateFactory, !!jupyterExtension)); - - context.subscriptions.push(serviceContainer.get(IInterpreterSelector)); - context.subscriptions.push(activateUpdateSparkLibraryProvider()); - - context.subscriptions.push(new ReplProvider(serviceContainer)); - context.subscriptions.push(new TerminalProvider(serviceContainer)); - context.subscriptions.push(new WorkspaceSymbols(serviceContainer)); - - type ConfigurationProvider = BaseConfigurationProvider; - serviceContainer.getAll(IDebugConfigurationProvider).forEach(debugConfig => { - context.subscriptions.push(debug.registerDebugConfigurationProvider(debugConfig.debugType, debugConfig)); - }); - activationDeferred.resolve(); -} - -function registerServices(context: ExtensionContext, serviceManager: ServiceManager, serviceContainer: ServiceContainer) { - serviceManager.addSingletonInstance(IServiceContainer, serviceContainer); - serviceManager.addSingletonInstance(IDisposableRegistry, context.subscriptions); - serviceManager.addSingletonInstance(IMemento, context.globalState, GLOBAL_MEMENTO); - serviceManager.addSingletonInstance(IMemento, context.workspaceState, WORKSPACE_MEMENTO); - - const standardOutputChannel = window.createOutputChannel('Python'); - const unitTestOutChannel = window.createOutputChannel('Python Test Log'); - serviceManager.addSingletonInstance(IOutputChannel, standardOutputChannel, STANDARD_OUTPUT_CHANNEL); - serviceManager.addSingletonInstance(IOutputChannel, unitTestOutChannel, TEST_OUTPUT_CHANNEL); - - commonRegisterTypes(serviceManager); - processRegisterTypes(serviceManager); - variableRegisterTypes(serviceManager); - unitTestsRegisterTypes(serviceManager); - lintersRegisterTypes(serviceManager); - interpretersRegisterTypes(serviceManager); - formattersRegisterTypes(serviceManager); - platformRegisterTypes(serviceManager); - installerRegisterTypes(serviceManager); - commonRegisterTerminalTypes(serviceManager); - debugConfigurationRegisterTypes(serviceManager); -} - -async function sendStartupTelemetry(activatedPromise: Promise, serviceContainer: IServiceContainer) { - const stopWatch = new StopWatch(); - const logger = serviceContainer.get(ILogger); - try { - await activatedPromise; - const duration = stopWatch.elapsedTime; - const condaLocator = serviceContainer.get(ICondaService); - const condaVersion = await condaLocator.getCondaVersion().catch(() => undefined); - const props = condaVersion ? { condaVersion } : undefined; - sendTelemetryEvent(EDITOR_LOAD, duration, props); - } catch (ex) { - logger.logError('sendStartupTelemetry failed.', ex); - } -} +'use strict'; +// This line should always be right on top. +// tslint:disable-next-line:no-any +if ((Reflect as any).metadata === undefined) { + // tslint:disable-next-line:no-require-imports no-var-requires + require('reflect-metadata'); +} +import { Container } from 'inversify'; +import { + debug, Disposable, ExtensionContext, + extensions, IndentAction, languages, Memento, + OutputChannel, window +} from 'vscode'; +import { AnalysisExtensionActivator } from './activation/analysis'; +import { ClassicExtensionActivator } from './activation/classic'; +import { IExtensionActivator } from './activation/types'; +import { PythonSettings } from './common/configSettings'; +import { isPythonAnalysisEngineTest, PYTHON, PYTHON_LANGUAGE, STANDARD_OUTPUT_CHANNEL } from './common/constants'; +import { FeatureDeprecationManager } from './common/featureDeprecationManager'; +import { createDeferred } from './common/helpers'; +import { PythonInstaller } from './common/installer/pythonInstallation'; +import { registerTypes as installerRegisterTypes } from './common/installer/serviceRegistry'; +import { registerTypes as platformRegisterTypes } from './common/platform/serviceRegistry'; +import { registerTypes as processRegisterTypes } from './common/process/serviceRegistry'; +import { registerTypes as commonRegisterTypes } from './common/serviceRegistry'; +import { StopWatch } from './common/stopWatch'; +import { GLOBAL_MEMENTO, IConfigurationService, IDisposableRegistry, ILogger, IMemento, IOutputChannel, IPersistentStateFactory, WORKSPACE_MEMENTO } from './common/types'; +import { registerTypes as variableRegisterTypes } from './common/variables/serviceRegistry'; +import { AttachRequestArguments, LaunchRequestArguments } from './debugger/Common/Contracts'; +import { BaseConfigurationProvider } from './debugger/configProviders/baseProvider'; +import { registerTypes as debugConfigurationRegisterTypes } from './debugger/configProviders/serviceRegistry'; +import { IDebugConfigurationProvider } from './debugger/types'; +import { registerTypes as formattersRegisterTypes } from './formatters/serviceRegistry'; +import { IInterpreterSelector } from './interpreter/configuration/types'; +import { ICondaService, IInterpreterService } from './interpreter/contracts'; +import { registerTypes as interpretersRegisterTypes } from './interpreter/serviceRegistry'; +import { ServiceContainer } from './ioc/container'; +import { ServiceManager } from './ioc/serviceManager'; +import { IServiceContainer } from './ioc/types'; +import { LinterCommands } from './linters/linterCommands'; +import { registerTypes as lintersRegisterTypes } from './linters/serviceRegistry'; +import { ILintingEngine } from './linters/types'; +import { DocStringFoldingProvider } from './providers/docStringFoldingProvider'; +import { PythonFormattingEditProvider } from './providers/formatProvider'; +import { LinterProvider } from './providers/linterProvider'; +import { ReplProvider } from './providers/replProvider'; +import { TerminalProvider } from './providers/terminalProvider'; +import { activateUpdateSparkLibraryProvider } from './providers/updateSparkLibraryProvider'; +import * as sortImports from './sortImports'; +import { sendTelemetryEvent } from './telemetry'; +import { EDITOR_LOAD } from './telemetry/constants'; +import { registerTypes as commonRegisterTerminalTypes } from './terminals/serviceRegistry'; +import { ICodeExecutionManager } from './terminals/types'; +import { BlockFormatProviders } from './typeFormatters/blockFormatProvider'; +import { OnEnterFormatter } from './typeFormatters/onEnterFormatter'; +import { TEST_OUTPUT_CHANNEL } from './unittests/common/constants'; +import { registerTypes as unitTestsRegisterTypes } from './unittests/serviceRegistry'; +import { WorkspaceSymbols } from './workspaceSymbols/main'; + +const activationDeferred = createDeferred(); +export const activated = activationDeferred.promise; + +// tslint:disable-next-line:max-func-body-length +export async function activate(context: ExtensionContext) { + const cont = new Container(); + const serviceManager = new ServiceManager(cont); + const serviceContainer = new ServiceContainer(cont); + registerServices(context, serviceManager, serviceContainer); + + const interpreterManager = serviceContainer.get(IInterpreterService); + // This must be completed before we can continue as language server needs the interpreter path. + interpreterManager.initialize(); + await interpreterManager.autoSetInterpreter(); + + const configuration = serviceManager.get(IConfigurationService); + const pythonSettings = configuration.getSettings(); + + const activator: IExtensionActivator = isPythonAnalysisEngineTest() || !pythonSettings.jediEnabled + ? new AnalysisExtensionActivator(serviceManager, pythonSettings) + : new ClassicExtensionActivator(serviceManager, pythonSettings, PYTHON); + + await activator.activate(context); + + const standardOutputChannel = serviceManager.get(IOutputChannel, STANDARD_OUTPUT_CHANNEL); + sortImports.activate(context, standardOutputChannel, serviceManager); + + serviceManager.get(ICodeExecutionManager).registerCommands(); + // tslint:disable-next-line:no-floating-promises + sendStartupTelemetry(activated, serviceContainer); + + const pythonInstaller = new PythonInstaller(serviceContainer); + pythonInstaller.checkPythonInstallation(PythonSettings.getInstance()) + .catch(ex => console.error('Python Extension: pythonInstaller.checkPythonInstallation', ex)); + + interpreterManager.refresh() + .catch(ex => console.error('Python Extension: interpreterManager.refresh', ex)); + + const jupyterExtension = extensions.getExtension('donjayamanne.jupyter'); + const lintingEngine = serviceManager.get(ILintingEngine); + lintingEngine.linkJupiterExtension(jupyterExtension).ignoreErrors(); + + context.subscriptions.push(new LinterCommands(serviceManager)); + const linterProvider = new LinterProvider(context, serviceManager); + context.subscriptions.push(linterProvider); + + // Enable indentAction + // tslint:disable-next-line:no-non-null-assertion + languages.setLanguageConfiguration(PYTHON_LANGUAGE, { + onEnterRules: [ + { + beforeText: /^\s*(?:def|class|for|if|elif|else|while|try|with|finally|except)\b.*:\s*\S+/, + action: { indentAction: IndentAction.None } + }, + { + beforeText: /^\s*(?:def|class|for|if|elif|else|while|try|with|finally|except|async)\b.*:\s*/, + action: { indentAction: IndentAction.Indent } + }, + { + beforeText: /^\s*#.*/, + afterText: /.+$/, + action: { indentAction: IndentAction.None, appendText: '# ' } + }, + { + beforeText: /^\s+(continue|break|return)\b.*/, + afterText: /\s+$/, + action: { indentAction: IndentAction.Outdent } + } + ] + }); + + if (pythonSettings && pythonSettings.formatting && pythonSettings.formatting.provider !== 'none') { + const formatProvider = new PythonFormattingEditProvider(context, serviceContainer); + context.subscriptions.push(languages.registerDocumentFormattingEditProvider(PYTHON, formatProvider)); + context.subscriptions.push(languages.registerDocumentRangeFormattingEditProvider(PYTHON, formatProvider)); + } + + context.subscriptions.push(languages.registerOnTypeFormattingEditProvider(PYTHON, new BlockFormatProviders(), ':')); + context.subscriptions.push(languages.registerOnTypeFormattingEditProvider(PYTHON, new OnEnterFormatter(), '\n')); + context.subscriptions.push(languages.registerFoldingRangeProvider(PYTHON, new DocStringFoldingProvider())); + + const persistentStateFactory = serviceManager.get(IPersistentStateFactory); + const deprecationMgr = new FeatureDeprecationManager(persistentStateFactory, !!jupyterExtension); + deprecationMgr.initialize(); + context.subscriptions.push(new FeatureDeprecationManager(persistentStateFactory, !!jupyterExtension)); + + context.subscriptions.push(serviceContainer.get(IInterpreterSelector)); + context.subscriptions.push(activateUpdateSparkLibraryProvider()); + + context.subscriptions.push(new ReplProvider(serviceContainer)); + context.subscriptions.push(new TerminalProvider(serviceContainer)); + context.subscriptions.push(new WorkspaceSymbols(serviceContainer)); + + type ConfigurationProvider = BaseConfigurationProvider; + serviceContainer.getAll(IDebugConfigurationProvider).forEach(debugConfig => { + context.subscriptions.push(debug.registerDebugConfigurationProvider(debugConfig.debugType, debugConfig)); + }); + activationDeferred.resolve(); +} + +function registerServices(context: ExtensionContext, serviceManager: ServiceManager, serviceContainer: ServiceContainer) { + serviceManager.addSingletonInstance(IServiceContainer, serviceContainer); + serviceManager.addSingletonInstance(IDisposableRegistry, context.subscriptions); + serviceManager.addSingletonInstance(IMemento, context.globalState, GLOBAL_MEMENTO); + serviceManager.addSingletonInstance(IMemento, context.workspaceState, WORKSPACE_MEMENTO); + + const standardOutputChannel = window.createOutputChannel('Python'); + const unitTestOutChannel = window.createOutputChannel('Python Test Log'); + serviceManager.addSingletonInstance(IOutputChannel, standardOutputChannel, STANDARD_OUTPUT_CHANNEL); + serviceManager.addSingletonInstance(IOutputChannel, unitTestOutChannel, TEST_OUTPUT_CHANNEL); + + commonRegisterTypes(serviceManager); + processRegisterTypes(serviceManager); + variableRegisterTypes(serviceManager); + unitTestsRegisterTypes(serviceManager); + lintersRegisterTypes(serviceManager); + interpretersRegisterTypes(serviceManager); + formattersRegisterTypes(serviceManager); + platformRegisterTypes(serviceManager); + installerRegisterTypes(serviceManager); + commonRegisterTerminalTypes(serviceManager); + debugConfigurationRegisterTypes(serviceManager); +} + +async function sendStartupTelemetry(activatedPromise: Promise, serviceContainer: IServiceContainer) { + const stopWatch = new StopWatch(); + const logger = serviceContainer.get(ILogger); + try { + await activatedPromise; + const duration = stopWatch.elapsedTime; + const condaLocator = serviceContainer.get(ICondaService); + const condaVersion = await condaLocator.getCondaVersion().catch(() => undefined); + const props = condaVersion ? { condaVersion } : undefined; + sendTelemetryEvent(EDITOR_LOAD, duration, props); + } catch (ex) { + logger.logError('sendStartupTelemetry failed.', ex); + } +} diff --git a/src/client/language/iterableTextRange.ts b/src/client/language/iterableTextRange.ts new file mode 100644 index 000000000000..6f92e1e769de --- /dev/null +++ b/src/client/language/iterableTextRange.ts @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +import { ITextRange, ITextRangeCollection } from './types'; + +export class IterableTextRange implements Iterable{ + constructor(private textRangeCollection: ITextRangeCollection) { + } + public [Symbol.iterator](): Iterator { + let index = -1; + + return { + next: (): IteratorResult => { + if (index < this.textRangeCollection.count - 1) { + return { + done: false, + value: this.textRangeCollection.getItemAt(index += 1) + }; + } else { + return { + done: true, + // tslint:disable-next-line:no-any + value: undefined as any + }; + } + } + }; + } +} diff --git a/src/client/providers/docStringFoldingProvider.ts b/src/client/providers/docStringFoldingProvider.ts new file mode 100644 index 000000000000..2b163cade5d1 --- /dev/null +++ b/src/client/providers/docStringFoldingProvider.ts @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +'use strict'; + +import { CancellationToken, FoldingContext, FoldingRange, FoldingRangeKind, FoldingRangeProvider, ProviderResult, Range, TextDocument } from 'vscode'; +import { IterableTextRange } from '../language/iterableTextRange'; +import { IToken, TokenizerMode, TokenType } from '../language/types'; +import { getDocumentTokens } from './providerUtilities'; + +export class DocStringFoldingProvider implements FoldingRangeProvider { + public provideFoldingRanges(document: TextDocument, _context: FoldingContext, token: CancellationToken): ProviderResult { + return this.getFoldingRanges(document); + } + + private getFoldingRanges(document: TextDocument) { + const tokenCollection = getDocumentTokens(document, document.lineAt(document.lineCount - 1).range.end, TokenizerMode.CommentsAndStrings); + const tokens = new IterableTextRange(tokenCollection); + + const docStringRanges: FoldingRange[] = []; + const commentRanges: FoldingRange[] = []; + + for (const token of tokens) { + const docstringRange = this.getDocStringFoldingRange(document, token); + if (docstringRange) { + docStringRanges.push(docstringRange); + continue; + } + + const commentRange = this.getSingleLineCommentRange(document, token); + if (commentRange) { + this.buildMultiLineCommentRange(commentRange, commentRanges); + } + } + + this.removeLastSingleLineComment(commentRanges); + return docStringRanges.concat(commentRanges); + } + private buildMultiLineCommentRange(commentRange: FoldingRange, commentRanges: FoldingRange[]) { + if (commentRanges.length === 0) { + commentRanges.push(commentRange); + return; + } + const previousComment = commentRanges[commentRanges.length - 1]; + if (previousComment.end + 1 === commentRange.start) { + previousComment.end = commentRange.end; + return; + } + if (previousComment.start === previousComment.end) { + commentRanges[commentRanges.length - 1] = commentRange; + return; + } + commentRanges.push(commentRange); + } + private removeLastSingleLineComment(commentRanges: FoldingRange[]) { + // Remove last comment folding range if its a single line entry. + if (commentRanges.length === 0) { + return; + } + const lastComment = commentRanges[commentRanges.length - 1]; + if (lastComment.start === lastComment.end) { + commentRanges.pop(); + } + } + private getDocStringFoldingRange(document: TextDocument, token: IToken) { + if (token.type !== TokenType.String) { + return; + } + + const startPosition = document.positionAt(token.start); + const endPosition = document.positionAt(token.end); + if (startPosition.line === endPosition.line) { + return; + } + + const startLine = document.lineAt(startPosition); + if (startLine.firstNonWhitespaceCharacterIndex !== startPosition.character) { + return; + } + const startIndex1 = startLine.text.indexOf('\'\'\''); + const startIndex2 = startLine.text.indexOf('"""'); + if (startIndex1 !== startPosition.character && startIndex2 !== startPosition.character) { + return; + } + + const range = new Range(startPosition, endPosition); + + return new FoldingRange(range.start.line, range.end.line); + } + private getSingleLineCommentRange(document: TextDocument, token: IToken) { + if (token.type !== TokenType.Comment) { + return; + } + + const startPosition = document.positionAt(token.start); + const endPosition = document.positionAt(token.end); + if (startPosition.line !== endPosition.line) { + return; + } + if (document.lineAt(startPosition).firstNonWhitespaceCharacterIndex !== startPosition.character) { + return; + } + + const range = new Range(startPosition, endPosition); + return new FoldingRange(range.start.line, range.end.line, FoldingRangeKind.Comment); + } +} diff --git a/src/test/providers/foldingProvider.test.ts b/src/test/providers/foldingProvider.test.ts new file mode 100644 index 000000000000..9de189afb47f --- /dev/null +++ b/src/test/providers/foldingProvider.test.ts @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { expect } from 'chai'; +import * as path from 'path'; +import { CancellationTokenSource, FoldingRange, FoldingRangeKind, workspace } from 'vscode'; +import { DocStringFoldingProvider } from '../../client/providers/docStringFoldingProvider'; + +type FileFoldingRanges = { file: string; ranges: FoldingRange[] }; +const pythonFilesPath = path.join(__dirname, '..', '..', '..', 'src', 'test', 'pythonFiles', 'folding'); + +// tslint:disable-next-line:max-func-body-length +suite('Provider - Folding Provider', () => { + const docStringFileAndExpectedFoldingRanges: FileFoldingRanges[] = [ + { + file: path.join(pythonFilesPath, 'attach_server.py'), ranges: [ + new FoldingRange(0, 14), new FoldingRange(44, 73, FoldingRangeKind.Comment), + new FoldingRange(95, 143), new FoldingRange(149, 150, FoldingRangeKind.Comment), + new FoldingRange(305, 313), new FoldingRange(320, 322) + ] + }, + { + file: path.join(pythonFilesPath, 'visualstudio_ipython_repl.py'), ranges: [ + new FoldingRange(0, 14), new FoldingRange(78, 79, FoldingRangeKind.Comment), + new FoldingRange(81, 82, FoldingRangeKind.Comment), new FoldingRange(92, 93, FoldingRangeKind.Comment), + new FoldingRange(108, 109, FoldingRangeKind.Comment), new FoldingRange(139, 140, FoldingRangeKind.Comment), + new FoldingRange(169, 170, FoldingRangeKind.Comment), new FoldingRange(275, 277, FoldingRangeKind.Comment), + new FoldingRange(319, 320, FoldingRangeKind.Comment) + ] + }, + { + file: path.join(pythonFilesPath, 'visualstudio_py_debugger.py'), ranges: [ + new FoldingRange(0, 15, FoldingRangeKind.Comment), new FoldingRange(22, 25, FoldingRangeKind.Comment), + new FoldingRange(47, 48, FoldingRangeKind.Comment), new FoldingRange(69, 70, FoldingRangeKind.Comment), + new FoldingRange(96, 97, FoldingRangeKind.Comment), new FoldingRange(105, 106, FoldingRangeKind.Comment), + new FoldingRange(141, 142, FoldingRangeKind.Comment), new FoldingRange(149, 162, FoldingRangeKind.Comment), + new FoldingRange(165, 166, FoldingRangeKind.Comment), new FoldingRange(207, 208, FoldingRangeKind.Comment), + new FoldingRange(235, 237, FoldingRangeKind.Comment), new FoldingRange(240, 241, FoldingRangeKind.Comment), + new FoldingRange(300, 301, FoldingRangeKind.Comment), new FoldingRange(334, 335, FoldingRangeKind.Comment), + new FoldingRange(346, 348, FoldingRangeKind.Comment), new FoldingRange(499, 500, FoldingRangeKind.Comment), + new FoldingRange(558, 559, FoldingRangeKind.Comment), new FoldingRange(602, 604, FoldingRangeKind.Comment), + new FoldingRange(608, 609, FoldingRangeKind.Comment), new FoldingRange(612, 614, FoldingRangeKind.Comment), + new FoldingRange(637, 638, FoldingRangeKind.Comment) + ] + }, + { + file: path.join(pythonFilesPath, 'visualstudio_py_repl.py'), ranges: [] + } + ]; + + docStringFileAndExpectedFoldingRanges.forEach(item => { + test(`Test Docstring folding regions '${path.basename(item.file)}'`, async () => { + const document = await workspace.openTextDocument(item.file); + const provider = new DocStringFoldingProvider(); + const ranges = await provider.provideFoldingRanges(document, {}, new CancellationTokenSource().token); + expect(ranges).to.be.lengthOf(item.ranges.length); + ranges!.forEach(range => { + const index = item.ranges + .findIndex(searchItem => searchItem.start === range.start && + searchItem.end === range.end); + expect(index).to.be.greaterThan(-1, `${range.start}, ${range.end} not found`); + }); + }); + }); +}); diff --git a/src/test/pythonFiles/folding/attach_server.py b/src/test/pythonFiles/folding/attach_server.py new file mode 100644 index 000000000000..c67dc9f106a6 --- /dev/null +++ b/src/test/pythonFiles/folding/attach_server.py @@ -0,0 +1,330 @@ +# Python Tools for Visual Studio +# Copyright(c) Microsoft Corporation +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the License); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +# OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +# IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABLITY OR NON-INFRINGEMENT. +# +# See the Apache Version 2.0 License for specific language governing +# permissions and limitations under the License. + +__author__ = "Microsoft Corporation " +__version__ = "3.0.0.0" + +__all__ = ['enable_attach', 'wait_for_attach', 'break_into_debugger', 'settrace', 'is_attached', 'AttachAlreadyEnabledError'] + +import atexit +import getpass +import os +import os.path +import platform +import socket +import struct +import sys +import threading +try: + import thread +except ImportError: + import _thread as thread +try: + import ssl +except ImportError: + ssl = None + +import ptvsd.visualstudio_py_debugger as vspd +import ptvsd.visualstudio_py_repl as vspr +from ptvsd.visualstudio_py_util import to_bytes, read_bytes, read_int, read_string, write_bytes, write_int, write_string + + +# The server (i.e. the Python app) waits on a TCP port provided. Whenever anything connects to that port, +# it immediately sends the octet sequence 'PTVSDBG', followed by version number represented as int64, +# and then waits for the client to respond with the same exact byte sequence. After signatures are thereby +# exchanged and found to match, the client is expected to provide a string secret (in the usual debugger +# string format, None/ACII/Unicode prefix + length + data), which can be an empty string to designate the +# lack of a specified secret. +# +# If the secret does not match the one expected by the server, it responds with 'RJCT', and then closes +# the connection. Otherwise, the server responds with 'ACPT', and awaits a 4-octet command. The following +# commands are recognized: +# +# 'INFO' +# Report information about the process. The server responds with the following information, in order: +# - Process ID (int64) +# - Executable name (string) +# - User name (string) +# - Implementation name (string) +# and then immediately closes connection. Note, all string fields can be empty or null strings. +# +# 'ATCH' +# Attach debugger to the process. If successful, the server responds with 'ACPT', followed by process ID +# (int64), and then the Python language version that the server is running represented by three int64s - +# major, minor, micro; From there on the socket is assumed to be using the normal PTVS debugging protocol. +# If attaching was not successful (which can happen if some other debugger is already attached), the server +# responds with 'RJCT' and closes the connection. +# +# 'REPL' +# Attach REPL to the process. If successful, the server responds with 'ACPT', and from there on the socket +# is assumed to be using the normal PTVS REPL protocol. If not successful (which can happen if there is +# no debugger attached), the server responds with 'RJCT' and closes the connection. + +PTVS_VER = '2.2' +DEFAULT_PORT = 5678 +PTVSDBG_VER = 6 # must be kept in sync with DebuggerProtocolVersion in PythonRemoteProcess.cs +PTVSDBG = to_bytes('PTVSDBG') +ACPT = to_bytes('ACPT') +RJCT = to_bytes('RJCT') +INFO = to_bytes('INFO') +ATCH = to_bytes('ATCH') +REPL = to_bytes('REPL') + +_attach_enabled = False +_attached = threading.Event() +vspd.DONT_DEBUG.append(os.path.normcase(__file__)) + + +class AttachAlreadyEnabledError(Exception): + """`ptvsd.enable_attach` has already been called in this process.""" + + +def enable_attach(secret, address = ('0.0.0.0', DEFAULT_PORT), certfile = None, keyfile = None, redirect_output = True): + """Enables Python Tools for Visual Studio to attach to this process remotely + to debug Python code. + + Parameters + ---------- + secret : str + Used to validate the clients - only those clients providing the valid + secret will be allowed to connect to this server. On client side, the + secret is prepended to the Qualifier string, separated from the + hostname by ``'@'``, e.g.: ``'secret@myhost.cloudapp.net:5678'``. If + secret is ``None``, there's no validation, and any client can connect + freely. + address : (str, int), optional + Specifies the interface and port on which the debugging server should + listen for TCP connections. It is in the same format as used for + regular sockets of the `socket.AF_INET` family, i.e. a tuple of + ``(hostname, port)``. On client side, the server is identified by the + Qualifier string in the usual ``'hostname:port'`` format, e.g.: + ``'myhost.cloudapp.net:5678'``. Default is ``('0.0.0.0', 5678)``. + certfile : str, optional + Used to enable SSL. If not specified, or if set to ``None``, the + connection between this program and the debugger will be unsecure, + and can be intercepted on the wire. If specified, the meaning of this + parameter is the same as for `ssl.wrap_socket`. + keyfile : str, optional + Used together with `certfile` when SSL is enabled. Its meaning is the + same as for ``ssl.wrap_socket``. + redirect_output : bool, optional + Specifies whether any output (on both `stdout` and `stderr`) produced + by this program should be sent to the debugger. Default is ``True``. + + Notes + ----- + This function returns immediately after setting up the debugging server, + and does not block program execution. If you need to block until debugger + is attached, call `ptvsd.wait_for_attach`. The debugger can be detached + and re-attached multiple times after `enable_attach` is called. + + This function can only be called once during the lifetime of the process. + On a second call, `AttachAlreadyEnabledError` is raised. In circumstances + where the caller does not control how many times the function will be + called (e.g. when a script with a single call is run more than once by + a hosting app or framework), the call should be wrapped in ``try..except``. + + Only the thread on which this function is called, and any threads that are + created after it returns, will be visible in the debugger once it is + attached. Any threads that are already running before this function is + called will not be visible. + """ + + if not ssl and (certfile or keyfile): + raise ValueError('could not import the ssl module - SSL is not supported on this version of Python') + + if sys.platform == 'cli': + # Check that IronPython was launched with -X:Frames and -X:Tracing, since we can't register our trace + # func on the thread that calls enable_attach otherwise + import clr + x_tracing = clr.GetCurrentRuntime().GetLanguageByExtension('py').Options.Tracing + x_frames = clr.GetCurrentRuntime().GetLanguageByExtension('py').Options.Frames + if not x_tracing or not x_frames: + raise RuntimeError('IronPython must be started with -X:Tracing and -X:Frames options to support PTVS remote debugging.') + + global _attach_enabled + if _attach_enabled: + raise AttachAlreadyEnabledError('ptvsd.enable_attach() has already been called in this process.') + _attach_enabled = True + + atexit.register(vspd.detach_process_and_notify_debugger) + + server = socket.socket(proto=socket.IPPROTO_TCP) + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(address) + server.listen(1) + def server_thread_func(): + while True: + client = None + raw_client = None + try: + client, addr = server.accept() + if certfile: + client = ssl.wrap_socket(client, server_side = True, ssl_version = ssl.PROTOCOL_TLSv1, certfile = certfile, keyfile = keyfile) + write_bytes(client, PTVSDBG) + write_int(client, PTVSDBG_VER) + + response = read_bytes(client, 7) + if response != PTVSDBG: + continue + dbg_ver = read_int(client) + if dbg_ver != PTVSDBG_VER: + continue + + client_secret = read_string(client) + if secret is None or secret == client_secret: + write_bytes(client, ACPT) + else: + write_bytes(client, RJCT) + continue + + response = read_bytes(client, 4) + + if response == INFO: + try: + pid = os.getpid() + except AttributeError: + pid = 0 + write_int(client, pid) + + exe = sys.executable or '' + write_string(client, exe) + + try: + username = getpass.getuser() + except AttributeError: + username = '' + write_string(client, username) + + try: + impl = platform.python_implementation() + except AttributeError: + try: + impl = sys.implementation.name + except AttributeError: + impl = 'Python' + + major, minor, micro, release_level, serial = sys.version_info + + os_and_arch = platform.system() + if os_and_arch == "": + os_and_arch = sys.platform + try: + if sys.maxsize > 2**32: + os_and_arch += ' 64-bit' + else: + os_and_arch += ' 32-bit' + except AttributeError: + pass + + version = '%s %s.%s.%s (%s)' % (impl, major, minor, micro, os_and_arch) + write_string(client, version) + + # Don't just drop the connection - let the debugger close it after it finishes reading. + client.recv(1) + + elif response == ATCH: + debug_options = vspd.parse_debug_options(read_string(client)) + if redirect_output: + debug_options.add('RedirectOutput') + + if vspd.DETACHED: + write_bytes(client, ACPT) + try: + pid = os.getpid() + except AttributeError: + pid = 0 + write_int(client, pid) + + major, minor, micro, release_level, serial = sys.version_info + write_int(client, major) + write_int(client, minor) + write_int(client, micro) + + vspd.attach_process_from_socket(client, debug_options, report = True) + vspd.mark_all_threads_for_break(vspd.STEPPING_ATTACH_BREAK) + _attached.set() + client = None + else: + write_bytes(client, RJCT) + + elif response == REPL: + if not vspd.DETACHED: + write_bytes(client, ACPT) + vspd.connect_repl_using_socket(client) + client = None + else: + write_bytes(client, RJCT) + + except (socket.error, OSError): + pass + finally: + if client is not None: + client.close() + + server_thread = threading.Thread(target = server_thread_func) + server_thread.setDaemon(True) + server_thread.start() + + frames = [] + f = sys._getframe() + while True: + f = f.f_back + if f is None: + break + frames.append(f) + frames.reverse() + cur_thread = vspd.new_thread() + for f in frames: + cur_thread.push_frame(f) + def replace_trace_func(): + for f in frames: + f.f_trace = cur_thread.trace_func + replace_trace_func() + sys.settrace(cur_thread.trace_func) + vspd.intercept_threads(for_attach = True) + + +# Alias for convenience of users of pydevd +settrace = enable_attach + + +def wait_for_attach(timeout = None): + """If a PTVS remote debugger is attached, returns immediately. Otherwise, + blocks until a remote debugger attaches to this process, or until the + optional timeout occurs. + + Parameters + ---------- + timeout : float, optional + The timeout for the operation in seconds (or fractions thereof). + """ + if vspd.DETACHED: + _attached.clear() + _attached.wait(timeout) + + +def break_into_debugger(): + """If a PTVS remote debugger is attached, pauses execution of all threads, + and breaks into the debugger with current thread as active. + """ + if not vspd.DETACHED: + vspd.SEND_BREAK_COMPLETE = thread.get_ident() + vspd.mark_all_threads_for_break() + +def is_attached(): + """Returns ``True`` if debugger is attached, ``False`` otherwise.""" + return not vspd.DETACHED diff --git a/src/test/pythonFiles/folding/empty.py b/src/test/pythonFiles/folding/empty.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/src/test/pythonFiles/folding/miscSamples.py b/src/test/pythonFiles/folding/miscSamples.py new file mode 100644 index 000000000000..01495fb0ee9c --- /dev/null +++ b/src/test/pythonFiles/folding/miscSamples.py @@ -0,0 +1,40 @@ + +def one(): + """comment""" + pass + +def two(): + value = """a doc string with single and double quotes "This is how it's done" """ + pass + +def three(): + """a doc string with single and double quotes "This is how it's done" + Another line + """ + pass + +def four(): + '''a doc string with single and double quotes "This is how it's done" ''' + pass + +def five(): + '''a doc string with single and double quotes "This is how it's done" + Another line + ''' + pass + +def six(): + """ s1 """ """ s2 """ + pass + +def seven(): + value = """ s1 """ """ s2 """ + pass + +def eight(): + ''' s1 ''' ''' s2 ''' + pass + +def nine(): + value = ''' s1 ''' ''' s2 ''' + pass diff --git a/src/test/pythonFiles/folding/noComments.py b/src/test/pythonFiles/folding/noComments.py new file mode 100644 index 000000000000..ca4d3f4140a6 --- /dev/null +++ b/src/test/pythonFiles/folding/noComments.py @@ -0,0 +1,278 @@ +__author__ = "Microsoft Corporation " +__version__ = "3.0.0.0" + +__all__ = ['enable_attach', 'wait_for_attach', 'break_into_debugger', 'settrace', 'is_attached', 'AttachAlreadyEnabledError'] + +import atexit +import getpass +import os +import os.path +import platform +import socket +import struct +import sys +import threading +try: + import thread +except ImportError: + import _thread as thread +try: + import ssl +except ImportError: + ssl = None + +import ptvsd.visualstudio_py_debugger as vspd +import ptvsd.visualstudio_py_repl as vspr +from ptvsd.visualstudio_py_util import to_bytes, read_bytes, read_int, read_string, write_bytes, write_int, write_string + +PTVS_VER = '2.2' +DEFAULT_PORT = 5678 +PTVSDBG_VER = 6 +PTVSDBG = to_bytes('PTVSDBG') +ACPT = to_bytes('ACPT') +RJCT = to_bytes('RJCT') +INFO = to_bytes('INFO') +ATCH = to_bytes('ATCH') +REPL = to_bytes('REPL') + +_attach_enabled = False +_attached = threading.Event() +vspd.DONT_DEBUG.append(os.path.normcase(__file__)) + + +class AttachAlreadyEnabledError(Exception): + """`ptvsd.enable_attach` has already been called in this process.""" + + +def enable_attach(secret, address = ('0.0.0.0', DEFAULT_PORT), certfile = None, keyfile = None, redirect_output = True): + """Enables Python Tools for Visual Studio to attach to this process remotely + to debug Python code. + + Parameters + ---------- + secret : str + Used to validate the clients - only those clients providing the valid + secret will be allowed to connect to this server. On client side, the + secret is prepended to the Qualifier string, separated from the + hostname by ``'@'``, e.g.: ``'secret@myhost.cloudapp.net:5678'``. If + secret is ``None``, there's no validation, and any client can connect + freely. + address : (str, int), optional + Specifies the interface and port on which the debugging server should + listen for TCP connections. It is in the same format as used for + regular sockets of the `socket.AF_INET` family, i.e. a tuple of + ``(hostname, port)``. On client side, the server is identified by the + Qualifier string in the usual ``'hostname:port'`` format, e.g.: + ``'myhost.cloudapp.net:5678'``. Default is ``('0.0.0.0', 5678)``. + certfile : str, optional + Used to enable SSL. If not specified, or if set to ``None``, the + connection between this program and the debugger will be unsecure, + and can be intercepted on the wire. If specified, the meaning of this + parameter is the same as for `ssl.wrap_socket`. + keyfile : str, optional + Used together with `certfile` when SSL is enabled. Its meaning is the + same as for ``ssl.wrap_socket``. + redirect_output : bool, optional + Specifies whether any output (on both `stdout` and `stderr`) produced + by this program should be sent to the debugger. Default is ``True``. + + Notes + ----- + This function returns immediately after setting up the debugging server, + and does not block program execution. If you need to block until debugger + is attached, call `ptvsd.wait_for_attach`. The debugger can be detached + and re-attached multiple times after `enable_attach` is called. + + This function can only be called once during the lifetime of the process. + On a second call, `AttachAlreadyEnabledError` is raised. In circumstances + where the caller does not control how many times the function will be + called (e.g. when a script with a single call is run more than once by + a hosting app or framework), the call should be wrapped in ``try..except``. + + Only the thread on which this function is called, and any threads that are + created after it returns, will be visible in the debugger once it is + attached. Any threads that are already running before this function is + called will not be visible. + """ + + if not ssl and (certfile or keyfile): + raise ValueError('could not import the ssl module - SSL is not supported on this version of Python') + + if sys.platform == 'cli': + import clr + x_tracing = clr.GetCurrentRuntime().GetLanguageByExtension('py').Options.Tracing + x_frames = clr.GetCurrentRuntime().GetLanguageByExtension('py').Options.Frames + if not x_tracing or not x_frames: + raise RuntimeError('IronPython must be started with -X:Tracing and -X:Frames options to support PTVS remote debugging.') + + global _attach_enabled + if _attach_enabled: + raise AttachAlreadyEnabledError('ptvsd.enable_attach() has already been called in this process.') + _attach_enabled = True + + atexit.register(vspd.detach_process_and_notify_debugger) + + server = socket.socket(proto=socket.IPPROTO_TCP) + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(address) + server.listen(1) + def server_thread_func(): + while True: + client = None + raw_client = None + try: + client, addr = server.accept() + if certfile: + client = ssl.wrap_socket(client, server_side = True, ssl_version = ssl.PROTOCOL_TLSv1, certfile = certfile, keyfile = keyfile) + write_bytes(client, PTVSDBG) + write_int(client, PTVSDBG_VER) + + response = read_bytes(client, 7) + if response != PTVSDBG: + continue + dbg_ver = read_int(client) + if dbg_ver != PTVSDBG_VER: + continue + + client_secret = read_string(client) + if secret is None or secret == client_secret: + write_bytes(client, ACPT) + else: + write_bytes(client, RJCT) + continue + + response = read_bytes(client, 4) + + if response == INFO: + try: + pid = os.getpid() + except AttributeError: + pid = 0 + write_int(client, pid) + + exe = sys.executable or '' + write_string(client, exe) + + try: + username = getpass.getuser() + except AttributeError: + username = '' + write_string(client, username) + + try: + impl = platform.python_implementation() + except AttributeError: + try: + impl = sys.implementation.name + except AttributeError: + impl = 'Python' + + major, minor, micro, release_level, serial = sys.version_info + + os_and_arch = platform.system() + if os_and_arch == "": + os_and_arch = sys.platform + try: + if sys.maxsize > 2**32: + os_and_arch += ' 64-bit' + else: + os_and_arch += ' 32-bit' + except AttributeError: + pass + + version = '%s %s.%s.%s (%s)' % (impl, major, minor, micro, os_and_arch) + write_string(client, version) + + client.recv(1) + + elif response == ATCH: + debug_options = vspd.parse_debug_options(read_string(client)) + if redirect_output: + debug_options.add('RedirectOutput') + + if vspd.DETACHED: + write_bytes(client, ACPT) + try: + pid = os.getpid() + except AttributeError: + pid = 0 + write_int(client, pid) + + major, minor, micro, release_level, serial = sys.version_info + write_int(client, major) + write_int(client, minor) + write_int(client, micro) + + vspd.attach_process_from_socket(client, debug_options, report = True) + vspd.mark_all_threads_for_break(vspd.STEPPING_ATTACH_BREAK) + _attached.set() + client = None + else: + write_bytes(client, RJCT) + + elif response == REPL: + if not vspd.DETACHED: + write_bytes(client, ACPT) + vspd.connect_repl_using_socket(client) + client = None + else: + write_bytes(client, RJCT) + + except (socket.error, OSError): + pass + finally: + if client is not None: + client.close() + + server_thread = threading.Thread(target = server_thread_func) + server_thread.setDaemon(True) + server_thread.start() + + frames = [] + f = sys._getframe() + while True: + f = f.f_back + if f is None: + break + frames.append(f) + frames.reverse() + cur_thread = vspd.new_thread() + for f in frames: + cur_thread.push_frame(f) + def replace_trace_func(): + for f in frames: + f.f_trace = cur_thread.trace_func + replace_trace_func() + sys.settrace(cur_thread.trace_func) + vspd.intercept_threads(for_attach = True) + + +settrace = enable_attach + + +def wait_for_attach(timeout = None): + """If a PTVS remote debugger is attached, returns immediately. Otherwise, + blocks until a remote debugger attaches to this process, or until the + optional timeout occurs. + + Parameters + ---------- + timeout : float, optional + The timeout for the operation in seconds (or fractions thereof). + """ + if vspd.DETACHED: + _attached.clear() + _attached.wait(timeout) + + +def break_into_debugger(): + """If a PTVS remote debugger is attached, pauses execution of all threads, + and breaks into the debugger with current thread as active. + """ + if not vspd.DETACHED: + vspd.SEND_BREAK_COMPLETE = thread.get_ident() + vspd.mark_all_threads_for_break() + +def is_attached(): + """Returns ``True`` if debugger is attached, ``False`` otherwise.""" + return not vspd.DETACHED diff --git a/src/test/pythonFiles/folding/noDocStrings.py b/src/test/pythonFiles/folding/noDocStrings.py new file mode 100644 index 000000000000..9fd4b4874a57 --- /dev/null +++ b/src/test/pythonFiles/folding/noDocStrings.py @@ -0,0 +1,266 @@ +# Python Tools for Visual Studio +# Copyright(c) Microsoft Corporation +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the License); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +# OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +# IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABLITY OR NON-INFRINGEMENT. +# +# See the Apache Version 2.0 License for specific language governing +# permissions and limitations under the License. + +__author__ = "Microsoft Corporation " +__version__ = "3.0.0.0" + +__all__ = ['enable_attach', 'wait_for_attach', 'break_into_debugger', 'settrace', 'is_attached', 'AttachAlreadyEnabledError'] + +import atexit +import getpass +import os +import os.path +import platform +import socket +import struct +import sys +import threading +try: + import thread +except ImportError: + import _thread as thread +try: + import ssl +except ImportError: + ssl = None + +import ptvsd.visualstudio_py_debugger as vspd +import ptvsd.visualstudio_py_repl as vspr +from ptvsd.visualstudio_py_util import to_bytes, read_bytes, read_int, read_string, write_bytes, write_int, write_string + + +# The server (i.e. the Python app) waits on a TCP port provided. Whenever anything connects to that port, +# it immediately sends the octet sequence 'PTVSDBG', followed by version number represented as int64, +# and then waits for the client to respond with the same exact byte sequence. After signatures are thereby +# exchanged and found to match, the client is expected to provide a string secret (in the usual debugger +# string format, None/ACII/Unicode prefix + length + data), which can be an empty string to designate the +# lack of a specified secret. +# +# If the secret does not match the one expected by the server, it responds with 'RJCT', and then closes +# the connection. Otherwise, the server responds with 'ACPT', and awaits a 4-octet command. The following +# commands are recognized: +# +# 'INFO' +# Report information about the process. The server responds with the following information, in order: +# - Process ID (int64) +# - Executable name (string) +# - User name (string) +# - Implementation name (string) +# and then immediately closes connection. Note, all string fields can be empty or null strings. +# +# 'ATCH' +# Attach debugger to the process. If successful, the server responds with 'ACPT', followed by process ID +# (int64), and then the Python language version that the server is running represented by three int64s - +# major, minor, micro; From there on the socket is assumed to be using the normal PTVS debugging protocol. +# If attaching was not successful (which can happen if some other debugger is already attached), the server +# responds with 'RJCT' and closes the connection. +# +# 'REPL' +# Attach REPL to the process. If successful, the server responds with 'ACPT', and from there on the socket +# is assumed to be using the normal PTVS REPL protocol. If not successful (which can happen if there is +# no debugger attached), the server responds with 'RJCT' and closes the connection. + +PTVS_VER = '2.2' +DEFAULT_PORT = 5678 +PTVSDBG_VER = 6 # must be kept in sync with DebuggerProtocolVersion in PythonRemoteProcess.cs +PTVSDBG = to_bytes('PTVSDBG') +ACPT = to_bytes('ACPT') +RJCT = to_bytes('RJCT') +INFO = to_bytes('INFO') +ATCH = to_bytes('ATCH') +REPL = to_bytes('REPL') + +_attach_enabled = False +_attached = threading.Event() +vspd.DONT_DEBUG.append(os.path.normcase(__file__)) + + +class AttachAlreadyEnabledError(Exception): + + +def enable_attach(secret, address = ('0.0.0.0', DEFAULT_PORT), certfile = None, keyfile = None, redirect_output = True): + if not ssl and (certfile or keyfile): + raise ValueError('could not import the ssl module - SSL is not supported on this version of Python') + + if sys.platform == 'cli': + # Check that IronPython was launched with -X:Frames and -X:Tracing, since we can't register our trace + # func on the thread that calls enable_attach otherwise + import clr + x_tracing = clr.GetCurrentRuntime().GetLanguageByExtension('py').Options.Tracing + x_frames = clr.GetCurrentRuntime().GetLanguageByExtension('py').Options.Frames + if not x_tracing or not x_frames: + raise RuntimeError('IronPython must be started with -X:Tracing and -X:Frames options to support PTVS remote debugging.') + + global _attach_enabled + if _attach_enabled: + raise AttachAlreadyEnabledError('ptvsd.enable_attach() has already been called in this process.') + _attach_enabled = True + + atexit.register(vspd.detach_process_and_notify_debugger) + + server = socket.socket(proto=socket.IPPROTO_TCP) + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(address) + server.listen(1) + def server_thread_func(): + while True: + client = None + raw_client = None + try: + client, addr = server.accept() + if certfile: + client = ssl.wrap_socket(client, server_side = True, ssl_version = ssl.PROTOCOL_TLSv1, certfile = certfile, keyfile = keyfile) + write_bytes(client, PTVSDBG) + write_int(client, PTVSDBG_VER) + + response = read_bytes(client, 7) + if response != PTVSDBG: + continue + dbg_ver = read_int(client) + if dbg_ver != PTVSDBG_VER: + continue + + client_secret = read_string(client) + if secret is None or secret == client_secret: + write_bytes(client, ACPT) + else: + write_bytes(client, RJCT) + continue + + response = read_bytes(client, 4) + + if response == INFO: + try: + pid = os.getpid() + except AttributeError: + pid = 0 + write_int(client, pid) + + exe = sys.executable or '' + write_string(client, exe) + + try: + username = getpass.getuser() + except AttributeError: + username = '' + write_string(client, username) + + try: + impl = platform.python_implementation() + except AttributeError: + try: + impl = sys.implementation.name + except AttributeError: + impl = 'Python' + + major, minor, micro, release_level, serial = sys.version_info + + os_and_arch = platform.system() + if os_and_arch == "": + os_and_arch = sys.platform + try: + if sys.maxsize > 2**32: + os_and_arch += ' 64-bit' + else: + os_and_arch += ' 32-bit' + except AttributeError: + pass + + version = '%s %s.%s.%s (%s)' % (impl, major, minor, micro, os_and_arch) + write_string(client, version) + + # Don't just drop the connection - let the debugger close it after it finishes reading. + client.recv(1) + + elif response == ATCH: + debug_options = vspd.parse_debug_options(read_string(client)) + if redirect_output: + debug_options.add('RedirectOutput') + + if vspd.DETACHED: + write_bytes(client, ACPT) + try: + pid = os.getpid() + except AttributeError: + pid = 0 + write_int(client, pid) + + major, minor, micro, release_level, serial = sys.version_info + write_int(client, major) + write_int(client, minor) + write_int(client, micro) + + vspd.attach_process_from_socket(client, debug_options, report = True) + vspd.mark_all_threads_for_break(vspd.STEPPING_ATTACH_BREAK) + _attached.set() + client = None + else: + write_bytes(client, RJCT) + + elif response == REPL: + if not vspd.DETACHED: + write_bytes(client, ACPT) + vspd.connect_repl_using_socket(client) + client = None + else: + write_bytes(client, RJCT) + + except (socket.error, OSError): + pass + finally: + if client is not None: + client.close() + + server_thread = threading.Thread(target = server_thread_func) + server_thread.setDaemon(True) + server_thread.start() + + frames = [] + f = sys._getframe() + while True: + f = f.f_back + if f is None: + break + frames.append(f) + frames.reverse() + cur_thread = vspd.new_thread() + for f in frames: + cur_thread.push_frame(f) + def replace_trace_func(): + for f in frames: + f.f_trace = cur_thread.trace_func + replace_trace_func() + sys.settrace(cur_thread.trace_func) + vspd.intercept_threads(for_attach = True) + + +# Alias for convenience of users of pydevd +settrace = enable_attach + + +def wait_for_attach(timeout = None): + if vspd.DETACHED: + _attached.clear() + _attached.wait(timeout) + + +def break_into_debugger(): + if not vspd.DETACHED: + vspd.SEND_BREAK_COMPLETE = thread.get_ident() + vspd.mark_all_threads_for_break() + +def is_attached(): + return not vspd.DETACHED diff --git a/src/test/pythonFiles/folding/visualstudio_ipython_repl.py b/src/test/pythonFiles/folding/visualstudio_ipython_repl.py new file mode 100644 index 000000000000..33aa109de971 --- /dev/null +++ b/src/test/pythonFiles/folding/visualstudio_ipython_repl.py @@ -0,0 +1,430 @@ +# Python Tools for Visual Studio +# Copyright(c) Microsoft Corporation +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the License); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +# OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +# IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABLITY OR NON-INFRINGEMENT. +# +# See the Apache Version 2.0 License for specific language governing +# permissions and limitations under the License. + +"""Implements REPL support over IPython/ZMQ for VisualStudio""" + +__author__ = "Microsoft Corporation " +__version__ = "3.0.0.0" + +import re +import sys +from visualstudio_py_repl import BasicReplBackend, ReplBackend, UnsupportedReplException, _command_line_to_args_list +from visualstudio_py_util import to_bytes +try: + import thread +except: + import _thread as thread # Renamed as Py3k + +from base64 import decodestring + +try: + import IPython +except ImportError: + exc_value = sys.exc_info()[1] + raise UnsupportedReplException('IPython mode requires IPython 0.11 or later: ' + str(exc_value)) + +def is_ipython_versionorgreater(major, minor): + """checks if we are at least a specific IPython version""" + match = re.match('(\d+).(\d+)', IPython.__version__) + if match: + groups = match.groups() + if int(groups[0]) > major: + return True + elif int(groups[0]) == major: + return int(groups[1]) >= minor + + return False + +remove_escapes = re.compile(r'\x1b[^m]*m') + +try: + if is_ipython_versionorgreater(3, 0): + from IPython.kernel import KernelManager + from IPython.kernel.channels import HBChannel + from IPython.kernel.threaded import (ThreadedZMQSocketChannel, ThreadedKernelClient as KernelClient) + ShellChannel = StdInChannel = IOPubChannel = ThreadedZMQSocketChannel + elif is_ipython_versionorgreater(1, 0): + from IPython.kernel import KernelManager, KernelClient + from IPython.kernel.channels import ShellChannel, HBChannel, StdInChannel, IOPubChannel + else: + import IPython.zmq + KernelClient = object # was split out from KernelManager in 1.0 + from IPython.zmq.kernelmanager import (KernelManager, + ShellSocketChannel as ShellChannel, + SubSocketChannel as IOPubChannel, + StdInSocketChannel as StdInChannel, + HBSocketChannel as HBChannel) + + from IPython.utils.traitlets import Type +except ImportError: + exc_value = sys.exc_info()[1] + raise UnsupportedReplException(str(exc_value)) + + +# TODO: SystemExit exceptions come back to us as strings, can we automatically exit when ones raised somehow? + +##### +# Channels which forward events + +# Description of the messaging protocol +# http://ipython.scipy.org/doc/manual/html/development/messaging.html + + +class DefaultHandler(object): + def unknown_command(self, content): + import pprint + print('unknown command ' + str(type(self))) + pprint.pprint(content) + + def call_handlers(self, msg): + # msg_type: + # execute_reply + msg_type = 'handle_' + msg['msg_type'] + + getattr(self, msg_type, self.unknown_command)(msg['content']) + +class VsShellChannel(DefaultHandler, ShellChannel): + + def handle_execute_reply(self, content): + # we could have a payload here... + payload = content['payload'] + + for item in payload: + data = item.get('data') + if data is not None: + try: + # Could be named km.sub_channel for very old IPython, but + # those versions should not put 'data' in this payload + write_data = self._vs_backend.km.iopub_channel.write_data + except AttributeError: + pass + else: + write_data(data) + continue + + output = item.get('text', None) + if output is not None: + self._vs_backend.write_stdout(output) + self._vs_backend.send_command_executed() + + def handle_inspect_reply(self, content): + self.handle_object_info_reply(content) + + def handle_object_info_reply(self, content): + self._vs_backend.object_info_reply = content + self._vs_backend.members_lock.release() + + def handle_complete_reply(self, content): + self._vs_backend.complete_reply = content + self._vs_backend.members_lock.release() + + def handle_kernel_info_reply(self, content): + self._vs_backend.write_stdout(content['banner']) + + +class VsIOPubChannel(DefaultHandler, IOPubChannel): + def call_handlers(self, msg): + # only output events from our session or no sessions + # https://pytools.codeplex.com/workitem/1622 + parent = msg.get('parent_header') + if not parent or parent.get('session') == self.session.session: + msg_type = 'handle_' + msg['msg_type'] + getattr(self, msg_type, self.unknown_command)(msg['content']) + + def handle_display_data(self, content): + # called when user calls display() + data = content.get('data', None) + + if data is not None: + self.write_data(data) + + def handle_stream(self, content): + stream_name = content['name'] + if is_ipython_versionorgreater(3, 0): + output = content['text'] + else: + output = content['data'] + if stream_name == 'stdout': + self._vs_backend.write_stdout(output) + elif stream_name == 'stderr': + self._vs_backend.write_stderr(output) + # TODO: stdin can show up here, do we echo that? + + def handle_execute_result(self, content): + self.handle_execute_output(content) + + def handle_execute_output(self, content): + # called when an expression statement is printed, we treat + # identical to stream output but it always goes to stdout + output = content['data'] + execution_count = content['execution_count'] + self._vs_backend.execution_count = execution_count + 1 + self._vs_backend.send_prompt( + '\r\nIn [%d]: ' % (execution_count + 1), + ' ' + ('.' * (len(str(execution_count + 1)) + 2)) + ': ', + allow_multiple_statements=True + ) + self.write_data(output, execution_count) + + def write_data(self, data, execution_count = None): + output_xaml = data.get('application/xaml+xml', None) + if output_xaml is not None: + try: + if isinstance(output_xaml, str) and sys.version_info[0] >= 3: + output_xaml = output_xaml.encode('ascii') + self._vs_backend.write_xaml(decodestring(output_xaml)) + self._vs_backend.write_stdout('\n') + return + except: + pass + + output_png = data.get('image/png', None) + if output_png is not None: + try: + if isinstance(output_png, str) and sys.version_info[0] >= 3: + output_png = output_png.encode('ascii') + self._vs_backend.write_png(decodestring(output_png)) + self._vs_backend.write_stdout('\n') + return + except: + pass + + output_str = data.get('text/plain', None) + if output_str is not None: + if execution_count is not None: + if '\n' in output_str: + output_str = '\n' + output_str + output_str = 'Out[' + str(execution_count) + ']: ' + output_str + + self._vs_backend.write_stdout(output_str) + self._vs_backend.write_stdout('\n') + return + + def handle_error(self, content): + # TODO: this includes escape sequences w/ color, we need to unescape that + ename = content['ename'] + evalue = content['evalue'] + tb = content['traceback'] + self._vs_backend.write_stderr('\n'.join(tb)) + self._vs_backend.write_stdout('\n') + + def handle_execute_input(self, content): + # just a rebroadcast of the command to be executed, can be ignored + self._vs_backend.execution_count += 1 + self._vs_backend.send_prompt( + '\r\nIn [%d]: ' % (self._vs_backend.execution_count), + ' ' + ('.' * (len(str(self._vs_backend.execution_count)) + 2)) + ': ', + allow_multiple_statements=True + ) + pass + + def handle_status(self, content): + pass + + # Backwards compat w/ 0.13 + handle_pyin = handle_execute_input + handle_pyout = handle_execute_output + handle_pyerr = handle_error + + +class VsStdInChannel(DefaultHandler, StdInChannel): + def handle_input_request(self, content): + # queue this to another thread so we don't block the channel + def read_and_respond(): + value = self._vs_backend.read_line() + + self.input(value) + + thread.start_new_thread(read_and_respond, ()) + + +class VsHBChannel(DefaultHandler, HBChannel): + pass + + +class VsKernelManager(KernelManager, KernelClient): + shell_channel_class = Type(VsShellChannel) + if is_ipython_versionorgreater(1, 0): + iopub_channel_class = Type(VsIOPubChannel) + else: + sub_channel_class = Type(VsIOPubChannel) + stdin_channel_class = Type(VsStdInChannel) + hb_channel_class = Type(VsHBChannel) + + +class IPythonBackend(ReplBackend): + def __init__(self, mod_name = '__main__', launch_file = None): + ReplBackend.__init__(self) + self.launch_file = launch_file + self.mod_name = mod_name + self.km = VsKernelManager() + + if is_ipython_versionorgreater(0, 13): + # http://pytools.codeplex.com/workitem/759 + # IPython stopped accepting the ipython flag and switched to launcher, the new + # default is what we want though. + self.km.start_kernel(**{'extra_arguments': self.get_extra_arguments()}) + else: + self.km.start_kernel(**{'ipython': True, 'extra_arguments': self.get_extra_arguments()}) + self.km.start_channels() + self.exit_lock = thread.allocate_lock() + self.exit_lock.acquire() # used as an event + self.members_lock = thread.allocate_lock() + self.members_lock.acquire() + + self.km.shell_channel._vs_backend = self + self.km.stdin_channel._vs_backend = self + if is_ipython_versionorgreater(1, 0): + self.km.iopub_channel._vs_backend = self + else: + self.km.sub_channel._vs_backend = self + self.km.hb_channel._vs_backend = self + self.execution_count = 1 + + def get_extra_arguments(self): + if sys.version <= '2.': + return [unicode('--pylab=inline')] + return ['--pylab=inline'] + + def execute_file_as_main(self, filename, arg_string): + f = open(filename, 'rb') + try: + contents = f.read().replace(to_bytes("\r\n"), to_bytes("\n")) + finally: + f.close() + args = [filename] + _command_line_to_args_list(arg_string) + code = ''' +import sys +sys.argv = %(args)r +__file__ = %(filename)r +del sys +exec(compile(%(contents)r, %(filename)r, 'exec')) +''' % {'filename' : filename, 'contents':contents, 'args': args} + + self.run_command(code, True) + + def execution_loop(self): + # we've got a bunch of threads setup for communication, we just block + # here until we're requested to exit. + self.send_prompt('\r\nIn [1]: ', ' ...: ', allow_multiple_statements=True) + self.exit_lock.acquire() + + def run_command(self, command, silent = False): + if is_ipython_versionorgreater(3, 0): + self.km.execute(command, silent) + else: + self.km.shell_channel.execute(command, silent) + + def execute_file_ex(self, filetype, filename, args): + if filetype == 'script': + self.execute_file_as_main(filename, args) + else: + raise NotImplementedError("Cannot execute %s file" % filetype) + + def exit_process(self): + self.exit_lock.release() + + def get_members(self, expression): + """returns a tuple of the type name, instance members, and type members""" + text = expression + '.' + if is_ipython_versionorgreater(3, 0): + self.km.complete(text) + else: + self.km.shell_channel.complete(text, text, 1) + + self.members_lock.acquire() + + reply = self.complete_reply + + res = {} + text_len = len(text) + for member in reply['matches']: + res[member[text_len:]] = 'object' + + return ('unknown', res, {}) + + def get_signatures(self, expression): + """returns doc, args, vargs, varkw, defaults.""" + + if is_ipython_versionorgreater(3, 0): + self.km.inspect(expression, None, 2) + else: + self.km.shell_channel.object_info(expression) + + self.members_lock.acquire() + + reply = self.object_info_reply + if is_ipython_versionorgreater(3, 0): + data = reply['data'] + text = data['text/plain'] + text = remove_escapes.sub('', text) + return [(text, (), None, None, [])] + else: + argspec = reply['argspec'] + defaults = argspec['defaults'] + if defaults is not None: + defaults = [repr(default) for default in defaults] + else: + defaults = [] + return [(reply['docstring'], argspec['args'], argspec['varargs'], argspec['varkw'], defaults)] + + def interrupt_main(self): + """aborts the current running command""" + self.km.interrupt_kernel() + + def set_current_module(self, module): + pass + + def get_module_names(self): + """returns a list of module names""" + return [] + + def flush(self): + pass + + def init_debugger(self): + from os import path + self.run_command(''' +def __visualstudio_debugger_init(): + import sys + sys.path.append(''' + repr(path.dirname(__file__)) + ''') + import visualstudio_py_debugger + new_thread = visualstudio_py_debugger.new_thread() + sys.settrace(new_thread.trace_func) + visualstudio_py_debugger.intercept_threads(True) + +__visualstudio_debugger_init() +del __visualstudio_debugger_init +''', True) + + def attach_process(self, port, debugger_id): + self.run_command(''' +def __visualstudio_debugger_attach(): + import visualstudio_py_debugger + + def do_detach(): + visualstudio_py_debugger.DETACH_CALLBACKS.remove(do_detach) + + visualstudio_py_debugger.DETACH_CALLBACKS.append(do_detach) + visualstudio_py_debugger.attach_process(''' + str(port) + ''', ''' + repr(debugger_id) + ''', report = True, block = True) + +__visualstudio_debugger_attach() +del __visualstudio_debugger_attach +''', True) + +class IPythonBackendWithoutPyLab(IPythonBackend): + def get_extra_arguments(self): + return [] \ No newline at end of file diff --git a/src/test/pythonFiles/folding/visualstudio_ipython_repl_double_quotes.py b/src/test/pythonFiles/folding/visualstudio_ipython_repl_double_quotes.py new file mode 100644 index 000000000000..473046639147 --- /dev/null +++ b/src/test/pythonFiles/folding/visualstudio_ipython_repl_double_quotes.py @@ -0,0 +1,430 @@ +# Python Tools for Visual Studio +# Copyright(c) Microsoft Corporation +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the License); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +# OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +# IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABLITY OR NON-INFRINGEMENT. +# +# See the Apache Version 2.0 License for specific language governing +# permissions and limitations under the License. + +"""Implements REPL support over IPython/ZMQ for VisualStudio""" + +__author__ = "Microsoft Corporation " +__version__ = "3.0.0.0" + +import re +import sys +from visualstudio_py_repl import BasicReplBackend, ReplBackend, UnsupportedReplException, _command_line_to_args_list +from visualstudio_py_util import to_bytes +try: + import thread +except: + import _thread as thread # Renamed as Py3k + +from base64 import decodestring + +try: + import IPython +except ImportError: + exc_value = sys.exc_info()[1] + raise UnsupportedReplException('IPython mode requires IPython 0.11 or later: ' + str(exc_value)) + +def is_ipython_versionorgreater(major, minor): + """checks if we are at least a specific IPython version""" + match = re.match('(\d+).(\d+)', IPython.__version__) + if match: + groups = match.groups() + if int(groups[0]) > major: + return True + elif int(groups[0]) == major: + return int(groups[1]) >= minor + + return False + +remove_escapes = re.compile(r'\x1b[^m]*m') + +try: + if is_ipython_versionorgreater(3, 0): + from IPython.kernel import KernelManager + from IPython.kernel.channels import HBChannel + from IPython.kernel.threaded import (ThreadedZMQSocketChannel, ThreadedKernelClient as KernelClient) + ShellChannel = StdInChannel = IOPubChannel = ThreadedZMQSocketChannel + elif is_ipython_versionorgreater(1, 0): + from IPython.kernel import KernelManager, KernelClient + from IPython.kernel.channels import ShellChannel, HBChannel, StdInChannel, IOPubChannel + else: + import IPython.zmq + KernelClient = object # was split out from KernelManager in 1.0 + from IPython.zmq.kernelmanager import (KernelManager, + ShellSocketChannel as ShellChannel, + SubSocketChannel as IOPubChannel, + StdInSocketChannel as StdInChannel, + HBSocketChannel as HBChannel) + + from IPython.utils.traitlets import Type +except ImportError: + exc_value = sys.exc_info()[1] + raise UnsupportedReplException(str(exc_value)) + + +# TODO: SystemExit exceptions come back to us as strings, can we automatically exit when ones raised somehow? + +##### +# Channels which forward events + +# Description of the messaging protocol +# http://ipython.scipy.org/doc/manual/html/development/messaging.html + + +class DefaultHandler(object): + def unknown_command(self, content): + import pprint + print('unknown command ' + str(type(self))) + pprint.pprint(content) + + def call_handlers(self, msg): + # msg_type: + # execute_reply + msg_type = 'handle_' + msg['msg_type'] + + getattr(self, msg_type, self.unknown_command)(msg['content']) + +class VsShellChannel(DefaultHandler, ShellChannel): + + def handle_execute_reply(self, content): + # we could have a payload here... + payload = content['payload'] + + for item in payload: + data = item.get('data') + if data is not None: + try: + # Could be named km.sub_channel for very old IPython, but + # those versions should not put 'data' in this payload + write_data = self._vs_backend.km.iopub_channel.write_data + except AttributeError: + pass + else: + write_data(data) + continue + + output = item.get('text', None) + if output is not None: + self._vs_backend.write_stdout(output) + self._vs_backend.send_command_executed() + + def handle_inspect_reply(self, content): + self.handle_object_info_reply(content) + + def handle_object_info_reply(self, content): + self._vs_backend.object_info_reply = content + self._vs_backend.members_lock.release() + + def handle_complete_reply(self, content): + self._vs_backend.complete_reply = content + self._vs_backend.members_lock.release() + + def handle_kernel_info_reply(self, content): + self._vs_backend.write_stdout(content['banner']) + + +class VsIOPubChannel(DefaultHandler, IOPubChannel): + def call_handlers(self, msg): + # only output events from our session or no sessions + # https://pytools.codeplex.com/workitem/1622 + parent = msg.get('parent_header') + if not parent or parent.get('session') == self.session.session: + msg_type = 'handle_' + msg['msg_type'] + getattr(self, msg_type, self.unknown_command)(msg['content']) + + def handle_display_data(self, content): + # called when user calls display() + data = content.get('data', None) + + if data is not None: + self.write_data(data) + + def handle_stream(self, content): + stream_name = content['name'] + if is_ipython_versionorgreater(3, 0): + output = content['text'] + else: + output = content['data'] + if stream_name == 'stdout': + self._vs_backend.write_stdout(output) + elif stream_name == 'stderr': + self._vs_backend.write_stderr(output) + # TODO: stdin can show up here, do we echo that? + + def handle_execute_result(self, content): + self.handle_execute_output(content) + + def handle_execute_output(self, content): + # called when an expression statement is printed, we treat + # identical to stream output but it always goes to stdout + output = content['data'] + execution_count = content['execution_count'] + self._vs_backend.execution_count = execution_count + 1 + self._vs_backend.send_prompt( + '\r\nIn [%d]: ' % (execution_count + 1), + ' ' + ('.' * (len(str(execution_count + 1)) + 2)) + ': ', + allow_multiple_statements=True + ) + self.write_data(output, execution_count) + + def write_data(self, data, execution_count = None): + output_xaml = data.get('application/xaml+xml', None) + if output_xaml is not None: + try: + if isinstance(output_xaml, str) and sys.version_info[0] >= 3: + output_xaml = output_xaml.encode('ascii') + self._vs_backend.write_xaml(decodestring(output_xaml)) + self._vs_backend.write_stdout('\n') + return + except: + pass + + output_png = data.get('image/png', None) + if output_png is not None: + try: + if isinstance(output_png, str) and sys.version_info[0] >= 3: + output_png = output_png.encode('ascii') + self._vs_backend.write_png(decodestring(output_png)) + self._vs_backend.write_stdout('\n') + return + except: + pass + + output_str = data.get('text/plain', None) + if output_str is not None: + if execution_count is not None: + if '\n' in output_str: + output_str = '\n' + output_str + output_str = 'Out[' + str(execution_count) + ']: ' + output_str + + self._vs_backend.write_stdout(output_str) + self._vs_backend.write_stdout('\n') + return + + def handle_error(self, content): + # TODO: this includes escape sequences w/ color, we need to unescape that + ename = content['ename'] + evalue = content['evalue'] + tb = content['traceback'] + self._vs_backend.write_stderr('\n'.join(tb)) + self._vs_backend.write_stdout('\n') + + def handle_execute_input(self, content): + # just a rebroadcast of the command to be executed, can be ignored + self._vs_backend.execution_count += 1 + self._vs_backend.send_prompt( + '\r\nIn [%d]: ' % (self._vs_backend.execution_count), + ' ' + ('.' * (len(str(self._vs_backend.execution_count)) + 2)) + ': ', + allow_multiple_statements=True + ) + pass + + def handle_status(self, content): + pass + + # Backwards compat w/ 0.13 + handle_pyin = handle_execute_input + handle_pyout = handle_execute_output + handle_pyerr = handle_error + + +class VsStdInChannel(DefaultHandler, StdInChannel): + def handle_input_request(self, content): + # queue this to another thread so we don't block the channel + def read_and_respond(): + value = self._vs_backend.read_line() + + self.input(value) + + thread.start_new_thread(read_and_respond, ()) + + +class VsHBChannel(DefaultHandler, HBChannel): + pass + + +class VsKernelManager(KernelManager, KernelClient): + shell_channel_class = Type(VsShellChannel) + if is_ipython_versionorgreater(1, 0): + iopub_channel_class = Type(VsIOPubChannel) + else: + sub_channel_class = Type(VsIOPubChannel) + stdin_channel_class = Type(VsStdInChannel) + hb_channel_class = Type(VsHBChannel) + + +class IPythonBackend(ReplBackend): + def __init__(self, mod_name = '__main__', launch_file = None): + ReplBackend.__init__(self) + self.launch_file = launch_file + self.mod_name = mod_name + self.km = VsKernelManager() + + if is_ipython_versionorgreater(0, 13): + # http://pytools.codeplex.com/workitem/759 + # IPython stopped accepting the ipython flag and switched to launcher, the new + # default is what we want though. + self.km.start_kernel(**{'extra_arguments': self.get_extra_arguments()}) + else: + self.km.start_kernel(**{'ipython': True, 'extra_arguments': self.get_extra_arguments()}) + self.km.start_channels() + self.exit_lock = thread.allocate_lock() + self.exit_lock.acquire() # used as an event + self.members_lock = thread.allocate_lock() + self.members_lock.acquire() + + self.km.shell_channel._vs_backend = self + self.km.stdin_channel._vs_backend = self + if is_ipython_versionorgreater(1, 0): + self.km.iopub_channel._vs_backend = self + else: + self.km.sub_channel._vs_backend = self + self.km.hb_channel._vs_backend = self + self.execution_count = 1 + + def get_extra_arguments(self): + if sys.version <= '2.': + return [unicode('--pylab=inline')] + return ['--pylab=inline'] + + def execute_file_as_main(self, filename, arg_string): + f = open(filename, 'rb') + try: + contents = f.read().replace(to_bytes("\r\n"), to_bytes("\n")) + finally: + f.close() + args = [filename] + _command_line_to_args_list(arg_string) + code = """ +import sys +sys.argv = %(args)r +__file__ = %(filename)r +del sys +exec(compile(%(contents)r, %(filename)r, 'exec')) +""" % {'filename' : filename, 'contents':contents, 'args': args} + + self.run_command(code, True) + + def execution_loop(self): + # we've got a bunch of threads setup for communication, we just block + # here until we're requested to exit. + self.send_prompt('\r\nIn [1]: ', ' ...: ', allow_multiple_statements=True) + self.exit_lock.acquire() + + def run_command(self, command, silent = False): + if is_ipython_versionorgreater(3, 0): + self.km.execute(command, silent) + else: + self.km.shell_channel.execute(command, silent) + + def execute_file_ex(self, filetype, filename, args): + if filetype == 'script': + self.execute_file_as_main(filename, args) + else: + raise NotImplementedError("Cannot execute %s file" % filetype) + + def exit_process(self): + self.exit_lock.release() + + def get_members(self, expression): + """returns a tuple of the type name, instance members, and type members""" + text = expression + '.' + if is_ipython_versionorgreater(3, 0): + self.km.complete(text) + else: + self.km.shell_channel.complete(text, text, 1) + + self.members_lock.acquire() + + reply = self.complete_reply + + res = {} + text_len = len(text) + for member in reply['matches']: + res[member[text_len:]] = 'object' + + return ('unknown', res, {}) + + def get_signatures(self, expression): + """returns doc, args, vargs, varkw, defaults.""" + + if is_ipython_versionorgreater(3, 0): + self.km.inspect(expression, None, 2) + else: + self.km.shell_channel.object_info(expression) + + self.members_lock.acquire() + + reply = self.object_info_reply + if is_ipython_versionorgreater(3, 0): + data = reply['data'] + text = data['text/plain'] + text = remove_escapes.sub('', text) + return [(text, (), None, None, [])] + else: + argspec = reply['argspec'] + defaults = argspec['defaults'] + if defaults is not None: + defaults = [repr(default) for default in defaults] + else: + defaults = [] + return [(reply['docstring'], argspec['args'], argspec['varargs'], argspec['varkw'], defaults)] + + def interrupt_main(self): + """aborts the current running command""" + self.km.interrupt_kernel() + + def set_current_module(self, module): + pass + + def get_module_names(self): + """returns a list of module names""" + return [] + + def flush(self): + pass + + def init_debugger(self): + from os import path + self.run_command(""" +def __visualstudio_debugger_init(): + import sys + sys.path.append(""" + repr(path.dirname(__file__)) + """) + import visualstudio_py_debugger + new_thread = visualstudio_py_debugger.new_thread() + sys.settrace(new_thread.trace_func) + visualstudio_py_debugger.intercept_threads(True) + +__visualstudio_debugger_init() +del __visualstudio_debugger_init +""", True) + + def attach_process(self, port, debugger_id): + self.run_command(""" +def __visualstudio_debugger_attach(): + import visualstudio_py_debugger + + def do_detach(): + visualstudio_py_debugger.DETACH_CALLBACKS.remove(do_detach) + + visualstudio_py_debugger.DETACH_CALLBACKS.append(do_detach) + visualstudio_py_debugger.attach_process(""" + str(port) + """, """ + repr(debugger_id) + """, report = True, block = True) + +__visualstudio_debugger_attach() +del __visualstudio_debugger_attach +""", True) + +class IPythonBackendWithoutPyLab(IPythonBackend): + def get_extra_arguments(self): + return [] diff --git a/src/test/pythonFiles/folding/visualstudio_py_debugger.py b/src/test/pythonFiles/folding/visualstudio_py_debugger.py new file mode 100644 index 000000000000..ec18ff8c63b0 --- /dev/null +++ b/src/test/pythonFiles/folding/visualstudio_py_debugger.py @@ -0,0 +1,644 @@ +# Python Tools for Visual Studio +# Copyright(c) Microsoft Corporation +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the License); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +# OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +# IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABLITY OR NON-INFRINGEMENT. +# +# See the Apache Version 2.0 License for specific language governing +# permissions and limitations under the License. +# With number of modifications by Don Jayamanne + +from __future__ import with_statement + +__author__ = "Microsoft Corporation " +__version__ = "3.0.0.0" + +# This module MUST NOT import threading in global scope. This is because in a direct (non-ptvsd) +# attach scenario, it is loaded on the injected debugger attach thread, and if threading module +# hasn't been loaded already, it will assume that the thread on which it is being loaded is the +# main thread. This will cause issues when the thread goes away after attach completes. +_threading = None + +import sys +import ctypes +try: + import thread +except ImportError: + import _thread as thread +import socket +import struct +import weakref +import traceback +import types +import bisect +from os import path +import ntpath +import runpy +import datetime +from codecs import BOM_UTF8 + +try: + # In the local attach scenario, visualstudio_py_util is injected into globals() + # by PyDebugAttach before loading this module, and cannot be imported. + _vspu = visualstudio_py_util +except: + try: + import visualstudio_py_util as _vspu + except ImportError: + import ptvsd.visualstudio_py_util as _vspu + +to_bytes = _vspu.to_bytes +exec_file = _vspu.exec_file +exec_module = _vspu.exec_module +exec_code = _vspu.exec_code +read_bytes = _vspu.read_bytes +read_int = _vspu.read_int +read_string = _vspu.read_string +write_bytes = _vspu.write_bytes +write_int = _vspu.write_int +write_string = _vspu.write_string +safe_repr = _vspu.SafeRepr() + +try: + # In the local attach scenario, visualstudio_py_repl is injected into globals() + # by PyDebugAttach before loading this module, and cannot be imported. + _vspr = visualstudio_py_repl +except: + try: + import visualstudio_py_repl as _vspr + except ImportError: + import ptvsd.visualstudio_py_repl as _vspr + +try: + import stackless +except ImportError: + stackless = None + +try: + xrange +except: + xrange = range + +if sys.platform == 'cli': + import clr + from System.Runtime.CompilerServices import ConditionalWeakTable + IPY_SEEN_MODULES = ConditionalWeakTable[object, object]() + +# Import encodings early to avoid import on the debugger thread, which may cause deadlock +from encodings import utf_8 + +# WARNING: Avoid imports beyond this point, specifically on the debugger thread, as this may cause +# deadlock where the debugger thread performs an import while a user thread has the import lock + +# save start_new_thread so we can call it later, we'll intercept others calls to it. + +debugger_dll_handle = None +DETACHED = True +def thread_creator(func, args, kwargs = {}, *extra_args): + if not isinstance(args, tuple): + # args is not a tuple. This may be because we have become bound to a + # class, which has offset our arguments by one. + if isinstance(kwargs, tuple): + func, args = args, kwargs + kwargs = extra_args[0] if len(extra_args) > 0 else {} + + return _start_new_thread(new_thread_wrapper, (func, args, kwargs)) + +_start_new_thread = thread.start_new_thread +THREADS = {} +THREADS_LOCK = thread.allocate_lock() +MODULES = [] + +BREAK_ON_SYSTEMEXIT_ZERO = False +DEBUG_STDLIB = False +DJANGO_DEBUG = False + +RICH_EXCEPTIONS = False +IGNORE_DJANGO_TEMPLATE_WARNINGS = False + +# Py3k compat - alias unicode to str +try: + unicode +except: + unicode = str + +# A value of a synthesized child. The string is passed through to the variable list, and type is not displayed at all. +class SynthesizedValue(object): + def __init__(self, repr_value='', len_value=None): + self.repr_value = repr_value + self.len_value = len_value + def __repr__(self): + return self.repr_value + def __len__(self): + return self.len_value + +# Specifies list of files not to debug. Can be extended by other modules +# (the REPL does this for $attach support and not stepping into the REPL). +DONT_DEBUG = [path.normcase(__file__), path.normcase(_vspu.__file__)] +if sys.version_info >= (3, 3): + DONT_DEBUG.append(path.normcase('')) +if sys.version_info >= (3, 5): + DONT_DEBUG.append(path.normcase('')) + +# Contains information about all breakpoints in the process. Keys are line numbers on which +# there are breakpoints in any file, and values are dicts. For every line number, the +# corresponding dict contains all the breakpoints that fall on that line. The keys in that +# dict are tuples of the form (filename, breakpoint_id), each entry representing a single +# breakpoint, and values are BreakpointInfo objects. +# +# For example, given the following breakpoints: +# +# 1. In 'main.py' at line 10. +# 2. In 'main.py' at line 20. +# 3. In 'module.py' at line 10. +# +# the contents of BREAKPOINTS would be: +# {10: {('main.py', 1): ..., ('module.py', 3): ...}, 20: {('main.py', 2): ... }} +BREAKPOINTS = {} + +# Contains information about all pending (i.e. not yet bound) breakpoints in the process. +# Elements are BreakpointInfo objects. +PENDING_BREAKPOINTS = set() + +# Must be in sync with enum PythonBreakpointConditionKind in PythonBreakpoint.cs +BREAKPOINT_CONDITION_ALWAYS = 0 +BREAKPOINT_CONDITION_WHEN_TRUE = 1 +BREAKPOINT_CONDITION_WHEN_CHANGED = 2 + +# Must be in sync with enum PythonBreakpointPassCountKind in PythonBreakpoint.cs +BREAKPOINT_PASS_COUNT_ALWAYS = 0 +BREAKPOINT_PASS_COUNT_EVERY = 1 +BREAKPOINT_PASS_COUNT_WHEN_EQUAL = 2 +BREAKPOINT_PASS_COUNT_WHEN_EQUAL_OR_GREATER = 3 + +## Begin modification by Don Jayamanne +DJANGO_VERSIONS_IDENTIFIED = False +IS_DJANGO18 = False +IS_DJANGO19 = False +IS_DJANGO19_OR_HIGHER = False + +try: + dict_contains = dict.has_key +except: + try: + #Py3k does not have has_key anymore, and older versions don't have __contains__ + dict_contains = dict.__contains__ + except: + try: + dict_contains = dict.has_key + except NameError: + def dict_contains(d, key): + return d.has_key(key) +## End modification by Don Jayamanne + +class BreakpointInfo(object): + __slots__ = [ + 'breakpoint_id', 'filename', 'lineno', 'condition_kind', 'condition', + 'pass_count_kind', 'pass_count', 'is_bound', 'last_condition_value', + 'hit_count' + ] + + # For "when changed" breakpoints, this is used as the initial value of last_condition_value, + # such that it is guaranteed to not compare equal to any other value that it will get later. + _DUMMY_LAST_VALUE = object() + + def __init__(self, breakpoint_id, filename, lineno, condition_kind, condition, pass_count_kind, pass_count): + self.breakpoint_id = breakpoint_id + self.filename = filename + self.lineno = lineno + self.condition_kind = condition_kind + self.condition = condition + self.pass_count_kind = pass_count_kind + self.pass_count = pass_count + self.is_bound = False + self.last_condition_value = BreakpointInfo._DUMMY_LAST_VALUE + self.hit_count = 0 + + @staticmethod + def find_by_id(breakpoint_id): + for line, bp_dict in BREAKPOINTS.items(): + for (filename, bp_id), bp in bp_dict.items(): + if bp_id == breakpoint_id: + return bp + return None + +# lock for calling .send on the socket +send_lock = thread.allocate_lock() + +class _SendLockContextManager(object): + """context manager for send lock. Handles both acquiring/releasing the + send lock as well as detaching the debugger if the remote process + is disconnected""" + + def __enter__(self): + # mark that we're about to do socket I/O so we won't deliver + # debug events when we're debugging the standard library + cur_thread = get_thread_from_id(thread.get_ident()) + if cur_thread is not None: + cur_thread.is_sending = True + + send_lock.acquire() + + def __exit__(self, exc_type, exc_value, tb): + send_lock.release() + + # start sending debug events again + cur_thread = get_thread_from_id(thread.get_ident()) + if cur_thread is not None: + cur_thread.is_sending = False + + if exc_type is not None: + detach_threads() + detach_process() + # swallow the exception, we're no longer debugging + return True + +_SendLockCtx = _SendLockContextManager() + +SEND_BREAK_COMPLETE = False + +STEPPING_OUT = -1 # first value, we decrement below this +STEPPING_NONE = 0 +STEPPING_BREAK = 1 +STEPPING_LAUNCH_BREAK = 2 +STEPPING_ATTACH_BREAK = 3 +STEPPING_INTO = 4 +STEPPING_OVER = 5 # last value, we increment past this. + +USER_STEPPING = (STEPPING_OUT, STEPPING_INTO, STEPPING_OVER) + +FRAME_KIND_NONE = 0 +FRAME_KIND_PYTHON = 1 +FRAME_KIND_DJANGO = 2 + +DJANGO_BUILTINS = {'True': True, 'False': False, 'None': None} + +PYTHON_EVALUATION_RESULT_REPR_KIND_NORMAL = 0 # regular repr and hex repr (if applicable) for the evaluation result; length is len(result) +PYTHON_EVALUATION_RESULT_REPR_KIND_RAW = 1 # repr is raw representation of the value - see TYPES_WITH_RAW_REPR; length is len(repr) +PYTHON_EVALUATION_RESULT_REPR_KIND_RAWLEN = 2 # same as above, but only the length is reported, not the actual value + +PYTHON_EVALUATION_RESULT_EXPANDABLE = 1 +PYTHON_EVALUATION_RESULT_METHOD_CALL = 2 +PYTHON_EVALUATION_RESULT_SIDE_EFFECTS = 4 +PYTHON_EVALUATION_RESULT_RAW = 8 +PYTHON_EVALUATION_RESULT_HAS_RAW_REPR = 16 + +# Don't show attributes of these types if they come from the class (assume they are methods). +METHOD_TYPES = ( + types.FunctionType, + types.MethodType, + types.BuiltinFunctionType, + type("".__repr__), # method-wrapper +) + +# repr() for these types can be used as input for eval() to get the original value. +# float is intentionally not included because it is not always round-trippable (e.g inf, nan). +TYPES_WITH_ROUND_TRIPPING_REPR = set((type(None), int, bool, str, unicode)) +if sys.version[0] == '3': + TYPES_WITH_ROUND_TRIPPING_REPR.add(bytes) +else: + TYPES_WITH_ROUND_TRIPPING_REPR.add(long) + +# repr() for these types can be used as input for eval() to get the original value, provided that the same is true for all their elements. +COLLECTION_TYPES_WITH_ROUND_TRIPPING_REPR = set((tuple, list, set, frozenset)) + +# eval(repr(x)), but optimized for common types for which it is known that result == x. +def eval_repr(x): + def is_repr_round_tripping(x): + # Do exact type checks here - subclasses can override __repr__. + if type(x) in TYPES_WITH_ROUND_TRIPPING_REPR: + return True + elif type(x) in COLLECTION_TYPES_WITH_ROUND_TRIPPING_REPR: + # All standard sequence types are round-trippable if their elements are. + return all((is_repr_round_tripping(item) for item in x)) + else: + return False + if is_repr_round_tripping(x): + return x + else: + return eval(repr(x), {}) + +# key is type, value is function producing the raw repr +TYPES_WITH_RAW_REPR = { + unicode: (lambda s: s) +} + +# bytearray is 2.6+ +try: + # getfilesystemencoding is used here because it effectively corresponds to the notion of "locale encoding": + # current ANSI codepage on Windows, LC_CTYPE on Linux, UTF-8 on OS X - which is exactly what we want. + TYPES_WITH_RAW_REPR[bytearray] = lambda b: b.decode(sys.getfilesystemencoding(), 'ignore') +except: + pass + +if sys.version[0] == '3': + TYPES_WITH_RAW_REPR[bytes] = TYPES_WITH_RAW_REPR[bytearray] +else: + TYPES_WITH_RAW_REPR[str] = TYPES_WITH_RAW_REPR[unicode] + +if sys.version[0] == '3': + # work around a crashing bug on CPython 3.x where they take a hard stack overflow + # we'll never see this exception but it'll allow us to keep our try/except handler + # the same across all versions of Python + class StackOverflowException(Exception): pass +else: + StackOverflowException = RuntimeError + +ASBR = to_bytes('ASBR') +SETL = to_bytes('SETL') +THRF = to_bytes('THRF') +DETC = to_bytes('DETC') +NEWT = to_bytes('NEWT') +EXTT = to_bytes('EXTT') +EXIT = to_bytes('EXIT') +EXCP = to_bytes('EXCP') +EXC2 = to_bytes('EXC2') +MODL = to_bytes('MODL') +STPD = to_bytes('STPD') +BRKS = to_bytes('BRKS') +BRKF = to_bytes('BRKF') +BRKH = to_bytes('BRKH') +BRKC = to_bytes('BRKC') +BKHC = to_bytes('BKHC') +LOAD = to_bytes('LOAD') +EXCE = to_bytes('EXCE') +EXCR = to_bytes('EXCR') +CHLD = to_bytes('CHLD') +OUTP = to_bytes('OUTP') +REQH = to_bytes('REQH') +LAST = to_bytes('LAST') + +def get_thread_from_id(id): + THREADS_LOCK.acquire() + try: + return THREADS.get(id) + finally: + THREADS_LOCK.release() + +def should_send_frame(frame): + return (frame is not None and + frame.f_code not in DEBUG_ENTRYPOINTS and + path.normcase(frame.f_code.co_filename) not in DONT_DEBUG) + +KNOWN_DIRECTORIES = set((None, '')) +KNOWN_ZIPS = set() + +def is_file_in_zip(filename): + parent, name = path.split(path.abspath(filename)) + if parent in KNOWN_DIRECTORIES: + return False + elif parent in KNOWN_ZIPS: + return True + elif path.isdir(parent): + KNOWN_DIRECTORIES.add(parent) + return False + else: + KNOWN_ZIPS.add(parent) + return True + +def lookup_builtin(name, frame): + try: + return frame.f_builtins.get(bits) + except: + # http://ironpython.codeplex.com/workitem/30908 + builtins = frame.f_globals['__builtins__'] + if not isinstance(builtins, dict): + builtins = builtins.__dict__ + return builtins.get(name) + +def lookup_local(frame, name): + bits = name.split('.') + obj = frame.f_locals.get(bits[0]) or frame.f_globals.get(bits[0]) or lookup_builtin(bits[0], frame) + bits.pop(0) + while bits and obj is not None and type(obj) is types.ModuleType: + obj = getattr(obj, bits.pop(0), None) + return obj + +if sys.version_info[0] >= 3: + _EXCEPTIONS_MODULE = 'builtins' +else: + _EXCEPTIONS_MODULE = 'exceptions' + +def get_exception_name(exc_type): + if exc_type.__module__ == _EXCEPTIONS_MODULE: + return exc_type.__name__ + else: + return exc_type.__module__ + '.' + exc_type.__name__ + +# These constants come from Visual Studio - enum_EXCEPTION_STATE +BREAK_MODE_NEVER = 0 +BREAK_MODE_ALWAYS = 1 +BREAK_MODE_UNHANDLED = 32 + +BREAK_TYPE_NONE = 0 +BREAK_TYPE_UNHANDLED = 1 +BREAK_TYPE_HANDLED = 2 + +class ExceptionBreakInfo(object): + BUILT_IN_HANDLERS = { + path.normcase(''): ((None, None, '*'),), + path.normcase('build\\bdist.win32\\egg\\pkg_resources.py'): ((None, None, '*'),), + path.normcase('build\\bdist.win-amd64\\egg\\pkg_resources.py'): ((None, None, '*'),), + } + + def __init__(self): + self.default_mode = BREAK_MODE_UNHANDLED + self.break_on = { } + self.handler_cache = dict(self.BUILT_IN_HANDLERS) + self.handler_lock = thread.allocate_lock() + self.add_exception('exceptions.IndexError', BREAK_MODE_NEVER) + self.add_exception('builtins.IndexError', BREAK_MODE_NEVER) + self.add_exception('exceptions.KeyError', BREAK_MODE_NEVER) + self.add_exception('builtins.KeyError', BREAK_MODE_NEVER) + self.add_exception('exceptions.AttributeError', BREAK_MODE_NEVER) + self.add_exception('builtins.AttributeError', BREAK_MODE_NEVER) + self.add_exception('exceptions.StopIteration', BREAK_MODE_NEVER) + self.add_exception('builtins.StopIteration', BREAK_MODE_NEVER) + self.add_exception('exceptions.GeneratorExit', BREAK_MODE_NEVER) + self.add_exception('builtins.GeneratorExit', BREAK_MODE_NEVER) + + def clear(self): + self.default_mode = BREAK_MODE_UNHANDLED + self.break_on.clear() + self.handler_cache = dict(self.BUILT_IN_HANDLERS) + + def should_break(self, thread, ex_type, ex_value, trace): + probe_stack() + name = get_exception_name(ex_type) + mode = self.break_on.get(name, self.default_mode) + break_type = BREAK_TYPE_NONE + if mode & BREAK_MODE_ALWAYS: + if self.is_handled(thread, ex_type, ex_value, trace): + break_type = BREAK_TYPE_HANDLED + else: + break_type = BREAK_TYPE_UNHANDLED + elif (mode & BREAK_MODE_UNHANDLED) and not self.is_handled(thread, ex_type, ex_value, trace): + break_type = BREAK_TYPE_UNHANDLED + + if break_type: + if issubclass(ex_type, SystemExit): + if not BREAK_ON_SYSTEMEXIT_ZERO: + if not ex_value or (isinstance(ex_value, SystemExit) and not ex_value.code): + break_type = BREAK_TYPE_NONE + + return break_type + + def is_handled(self, thread, ex_type, ex_value, trace): + if trace is None: + # get out if we didn't get a traceback + return False + + if trace.tb_next is not None: + if should_send_frame(trace.tb_next.tb_frame) and should_debug_code(trace.tb_next.tb_frame.f_code): + # don't break if this is not the top of the traceback, + # unless the previous frame was not debuggable + return True + + cur_frame = trace.tb_frame + + while should_send_frame(cur_frame) and cur_frame.f_code is not None and cur_frame.f_code.co_filename is not None: + filename = path.normcase(cur_frame.f_code.co_filename) + if is_file_in_zip(filename): + # File is in a zip, so assume it handles exceptions + return True + + if not is_same_py_file(filename, __file__): + handlers = self.handler_cache.get(filename) + + if handlers is None: + # req handlers for this file from the debug engine + self.handler_lock.acquire() + + with _SendLockCtx: + write_bytes(conn, REQH) + write_string(conn, filename) + + # wait for the handler data to be received + self.handler_lock.acquire() + self.handler_lock.release() + + handlers = self.handler_cache.get(filename) + + if handlers is None: + # no code available, so assume unhandled + return False + + line = cur_frame.f_lineno + for line_start, line_end, expressions in handlers: + if line_start is None or line_start <= line < line_end: + if '*' in expressions: + return True + + for text in expressions: + try: + res = lookup_local(cur_frame, text) + if res is not None and issubclass(ex_type, res): + return True + except: + pass + + cur_frame = cur_frame.f_back + + return False + + def add_exception(self, name, mode=BREAK_MODE_UNHANDLED): + if name.startswith(_EXCEPTIONS_MODULE + '.'): + name = name[len(_EXCEPTIONS_MODULE) + 1:] + self.break_on[name] = mode + +BREAK_ON = ExceptionBreakInfo() + +def probe_stack(depth = 10): + """helper to make sure we have enough stack space to proceed w/o corrupting + debugger state.""" + if depth == 0: + return + probe_stack(depth - 1) + +PREFIXES = [path.normcase(sys.prefix)] +# If we're running in a virtual env, DEBUG_STDLIB should respect this too. +if hasattr(sys, 'base_prefix'): + PREFIXES.append(path.normcase(sys.base_prefix)) +if hasattr(sys, 'real_prefix'): + PREFIXES.append(path.normcase(sys.real_prefix)) + +def should_debug_code(code): + if not code or not code.co_filename: + return False + + filename = path.normcase(code.co_filename) + if not DEBUG_STDLIB: + for prefix in PREFIXES: + if prefix != '' and filename.startswith(prefix): + return False + + for dont_debug_file in DONT_DEBUG: + if is_same_py_file(filename, dont_debug_file): + return False + + if is_file_in_zip(filename): + # file in inside an egg or zip, so we can't debug it + return False + + return True + +attach_lock = thread.allocate() +attach_sent_break = False + +local_path_to_vs_path = {} + +def breakpoint_path_match(vs_path, local_path): + vs_path_norm = path.normcase(vs_path) + local_path_norm = path.normcase(local_path) + if local_path_to_vs_path.get(local_path_norm) == vs_path_norm: + return True + + # Walk the local filesystem from local_path up, matching agains win_path component by component, + # and stop when we no longer see an __init__.py. This should give a reasonably close approximation + # of matching the package name. + while True: + local_path, local_name = path.split(local_path) + vs_path, vs_name = ntpath.split(vs_path) + # Match the last component in the path. If one or both components are unavailable, then + # we have reached the root on the corresponding path without successfully matching. + if not local_name or not vs_name or path.normcase(local_name) != path.normcase(vs_name): + return False + # If we have an __init__.py, this module was inside the package, and we still need to match + # thatpackage, so walk up one level and keep matching. Otherwise, we've walked as far as we + # needed to, and matched all names on our way, so this is a match. + if not path.exists(path.join(local_path, '__init__.py')): + break + + local_path_to_vs_path[local_path_norm] = vs_path_norm + return True + +def update_all_thread_stacks(blocking_thread = None, check_is_blocked = True): + THREADS_LOCK.acquire() + all_threads = list(THREADS.values()) + THREADS_LOCK.release() + + for cur_thread in all_threads: + if cur_thread is blocking_thread: + continue + + cur_thread._block_starting_lock.acquire() + if not check_is_blocked or not cur_thread._is_blocked: + # release the lock, we're going to run user code to evaluate the frames + cur_thread._block_starting_lock.release() + + frames = cur_thread.get_frame_list() + + # re-acquire the lock and make sure we're still not blocked. If so send + # the frame list. + cur_thread._block_starting_lock.acquire() + if not check_is_blocked or not cur_thread._is_blocked: + cur_thread.send_frame_list(frames) + + cur_thread._block_starting_lock.release() diff --git a/src/test/pythonFiles/folding/visualstudio_py_repl.py b/src/test/pythonFiles/folding/visualstudio_py_repl.py new file mode 100644 index 000000000000..14259db2e30e --- /dev/null +++ b/src/test/pythonFiles/folding/visualstudio_py_repl.py @@ -0,0 +1,513 @@ +# Python Tools for Visual Studio + +# Copyright(c) Microsoft Corporation + +# All rights reserved. + +from __future__ import with_statement + +__author__ = "Microsoft Corporation " +__version__ = "3.0.0.0" + +# This module MUST NOT import threading in global scope. This is because in a direct (non-ptvsd) + +# attach scenario, it is loaded on the injected debugger attach thread, and if threading module + +# hasn't been loaded already, it will assume that the thread on which it is being loaded is the + +# main thread. This will cause issues when the thread goes away after attach completes. + +try: + import thread +except ImportError: + # Renamed in Python3k + import _thread as thread +try: + from ssl import SSLError +except: + SSLError = None + +import sys +import socket +import select +import time +import struct +import imp +import traceback +import random +import os +import inspect +import types +from collections import deque + +try: + # In the local attach scenario, visualstudio_py_util is injected into globals() + + # by PyDebugAttach before loading this module, and cannot be imported. + _vspu = visualstudio_py_util +except: + try: + import visualstudio_py_util as _vspu + except ImportError: + import ptvsd.visualstudio_py_util as _vspu +to_bytes = _vspu.to_bytes +read_bytes = _vspu.read_bytes +read_int = _vspu.read_int +read_string = _vspu.read_string +write_bytes = _vspu.write_bytes +write_int = _vspu.write_int +write_string = _vspu.write_string + +try: + unicode +except NameError: + unicode = str + +try: + BaseException +except NameError: + # BaseException not defined until Python 2.5 + BaseException = Exception + +DEBUG = os.environ.get('DEBUG_REPL') is not None + +__all__ = ['ReplBackend', 'BasicReplBackend', 'BACKEND'] + +def _debug_write(out): + if DEBUG: + sys.__stdout__.write(out) + sys.__stdout__.flush() + + +class SafeSendLock(object): + """a lock which ensures we're released if we take a KeyboardInterrupt exception acquiring it""" + def __init__(self): + self.lock = thread.allocate_lock() + + def __enter__(self): + self.acquire() + + def __exit__(self, exc_type, exc_value, tb): + self.release() + + def acquire(self): + try: + self.lock.acquire() + except KeyboardInterrupt: + try: + self.lock.release() + except: + pass + raise + + def release(self): + self.lock.release() + +def _command_line_to_args_list(cmdline): + """splits a string into a list using Windows command line syntax.""" + args_list = [] + + if cmdline and cmdline.strip(): + from ctypes import c_int, c_voidp, c_wchar_p + from ctypes import byref, POINTER, WinDLL + + clta = WinDLL('shell32').CommandLineToArgvW + clta.argtypes = [c_wchar_p, POINTER(c_int)] + clta.restype = POINTER(c_wchar_p) + + lf = WinDLL('kernel32').LocalFree + lf.argtypes = [c_voidp] + + pNumArgs = c_int() + r = clta(cmdline, byref(pNumArgs)) + if r: + for index in range(0, pNumArgs.value): + if sys.hexversion >= 0x030000F0: + argval = r[index] + else: + argval = r[index].encode('ascii', 'replace') + args_list.append(argval) + lf(r) + else: + sys.stderr.write('Error parsing script arguments:\n') + sys.stderr.write(cmdline + '\n') + + return args_list + + +class UnsupportedReplException(Exception): + def __init__(self, reason): + self.reason = reason + +# save the start_new_thread so we won't debug/break into the REPL comm thread. +start_new_thread = thread.start_new_thread +class ReplBackend(object): + """back end for executing REPL code. This base class handles all of the communication with the remote process while derived classes implement the actual inspection and introspection.""" + _MRES = to_bytes('MRES') + _SRES = to_bytes('SRES') + _MODS = to_bytes('MODS') + _IMGD = to_bytes('IMGD') + _PRPC = to_bytes('PRPC') + _RDLN = to_bytes('RDLN') + _STDO = to_bytes('STDO') + _STDE = to_bytes('STDE') + _DBGA = to_bytes('DBGA') + _DETC = to_bytes('DETC') + _DPNG = to_bytes('DPNG') + _DXAM = to_bytes('DXAM') + _CHWD = to_bytes('CHWD') + + _MERR = to_bytes('MERR') + _SERR = to_bytes('SERR') + _ERRE = to_bytes('ERRE') + _EXIT = to_bytes('EXIT') + _DONE = to_bytes('DONE') + _MODC = to_bytes('MODC') + + def __init__(self, *args, **kwargs): + import threading + self.conn = None + self.send_lock = SafeSendLock() + self.input_event = threading.Lock() + self.input_event.acquire() # lock starts acquired (we use it like a manual reset event) + self.input_string = None + self.exit_requested = False + + def connect(self, port): + self.conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.conn.connect(('127.0.0.1', port)) + + # start a new thread for communicating w/ the remote process + start_new_thread(self._repl_loop, ()) + + def connect_using_socket(self, socket): + self.conn = socket + start_new_thread(self._repl_loop, ()) + + def _repl_loop(self): + """loop on created thread which processes communicates with the REPL window""" + try: + while True: + if self.check_for_exit_repl_loop(): + break + + # we receive a series of 4 byte commands. Each command then + + # has it's own format which we must parse before continuing to + + # the next command. + self.flush() + self.conn.settimeout(10) + + # 2.x raises SSLError in case of timeout (http://bugs.python.org/issue10272) + if SSLError: + timeout_exc_types = (socket.timeout, SSLError) + else: + timeout_exc_types = socket.timeout + try: + inp = read_bytes(self.conn, 4) + except timeout_exc_types: + r, w, x = select.select([], [], [self.conn], 0) + if x: + # an exception event has occured on the socket... + raise + continue + + self.conn.settimeout(None) + if inp == '': + break + self.flush() + + cmd = ReplBackend._COMMANDS.get(inp) + if cmd is not None: + cmd(self) + except: + _debug_write('error in repl loop') + _debug_write(traceback.format_exc()) + self.exit_process() + + time.sleep(2) # try and exit gracefully, then interrupt main if necessary + + if sys.platform == 'cli': + # just kill us as fast as possible + import System + System.Environment.Exit(1) + + self.interrupt_main() + + def check_for_exit_repl_loop(self): + return False + + def _cmd_run(self): + """runs the received snippet of code""" + self.run_command(read_string(self.conn)) + + def _cmd_abrt(self): + """aborts the current running command""" + # abort command, interrupts execution of the main thread. + self.interrupt_main() + + def _cmd_exit(self): + """exits the interactive process""" + self.exit_requested = True + self.exit_process() + + def _cmd_mems(self): + """gets the list of members available for the given expression""" + expression = read_string(self.conn) + try: + name, inst_members, type_members = self.get_members(expression) + except: + with self.send_lock: + write_bytes(self.conn, ReplBackend._MERR) + _debug_write('error in eval') + _debug_write(traceback.format_exc()) + else: + with self.send_lock: + write_bytes(self.conn, ReplBackend._MRES) + write_string(self.conn, name) + self._write_member_dict(inst_members) + self._write_member_dict(type_members) + + def _cmd_sigs(self): + """gets the signatures for the given expression""" + expression = read_string(self.conn) + try: + sigs = self.get_signatures(expression) + except: + with self.send_lock: + write_bytes(self.conn, ReplBackend._SERR) + _debug_write('error in eval') + _debug_write(traceback.format_exc()) + else: + with self.send_lock: + write_bytes(self.conn, ReplBackend._SRES) + # single overload + write_int(self.conn, len(sigs)) + for doc, args, vargs, varkw, defaults in sigs: + # write overload + write_string(self.conn, (doc or '')[:4096]) + arg_count = len(args) + (vargs is not None) + (varkw is not None) + write_int(self.conn, arg_count) + + def_values = [''] * (len(args) - len(defaults)) + ['=' + d for d in defaults] + for arg, def_value in zip(args, def_values): + write_string(self.conn, (arg or '') + def_value) + if vargs is not None: + write_string(self.conn, '*' + vargs) + if varkw is not None: + write_string(self.conn, '**' + varkw) + + def _cmd_setm(self): + global exec_mod + """sets the current module which code will execute against""" + mod_name = read_string(self.conn) + self.set_current_module(mod_name) + + def _cmd_sett(self): + """sets the current thread and frame which code will execute against""" + thread_id = read_int(self.conn) + frame_id = read_int(self.conn) + frame_kind = read_int(self.conn) + self.set_current_thread_and_frame(thread_id, frame_id, frame_kind) + + def _cmd_mods(self): + """gets the list of available modules""" + try: + res = self.get_module_names() + res.sort() + except: + res = [] + + with self.send_lock: + write_bytes(self.conn, ReplBackend._MODS) + write_int(self.conn, len(res)) + for name, filename in res: + write_string(self.conn, name) + write_string(self.conn, filename) + + def _cmd_inpl(self): + """handles the input command which returns a string of input""" + self.input_string = read_string(self.conn) + self.input_event.release() + + def _cmd_excf(self): + """handles executing a single file""" + filename = read_string(self.conn) + args = read_string(self.conn) + self.execute_file(filename, args) + + def _cmd_excx(self): + """handles executing a single file, module or process""" + filetype = read_string(self.conn) + filename = read_string(self.conn) + args = read_string(self.conn) + self.execute_file_ex(filetype, filename, args) + + def _cmd_debug_attach(self): + import visualstudio_py_debugger + port = read_int(self.conn) + id = read_string(self.conn) + debug_options = visualstudio_py_debugger.parse_debug_options(read_string(self.conn)) + self.attach_process(port, id, debug_options) + + _COMMANDS = { + to_bytes('run '): _cmd_run, + to_bytes('abrt'): _cmd_abrt, + to_bytes('exit'): _cmd_exit, + to_bytes('mems'): _cmd_mems, + to_bytes('sigs'): _cmd_sigs, + to_bytes('mods'): _cmd_mods, + to_bytes('setm'): _cmd_setm, + to_bytes('sett'): _cmd_sett, + to_bytes('inpl'): _cmd_inpl, + to_bytes('excf'): _cmd_excf, + to_bytes('excx'): _cmd_excx, + to_bytes('dbga'): _cmd_debug_attach, + } + + def _write_member_dict(self, mem_dict): + write_int(self.conn, len(mem_dict)) + for name, type_name in mem_dict.items(): + write_string(self.conn, name) + write_string(self.conn, type_name) + + def on_debugger_detach(self): + with self.send_lock: + write_bytes(self.conn, ReplBackend._DETC) + + def init_debugger(self): + from os import path + sys.path.append(path.dirname(__file__)) + import visualstudio_py_debugger + visualstudio_py_debugger.DONT_DEBUG.append(path.normcase(__file__)) + new_thread = visualstudio_py_debugger.new_thread() + sys.settrace(new_thread.trace_func) + visualstudio_py_debugger.intercept_threads(True) + + def send_image(self, filename): + with self.send_lock: + write_bytes(self.conn, ReplBackend._IMGD) + write_string(self.conn, filename) + + def write_png(self, image_bytes): + with self.send_lock: + write_bytes(self.conn, ReplBackend._DPNG) + write_int(self.conn, len(image_bytes)) + write_bytes(self.conn, image_bytes) + + def write_xaml(self, xaml_bytes): + with self.send_lock: + write_bytes(self.conn, ReplBackend._DXAM) + write_int(self.conn, len(xaml_bytes)) + write_bytes(self.conn, xaml_bytes) + + def send_prompt(self, ps1, ps2, allow_multiple_statements): + """sends the current prompt to the interactive window""" + with self.send_lock: + write_bytes(self.conn, ReplBackend._PRPC) + write_string(self.conn, ps1) + write_string(self.conn, ps2) + write_int(self.conn, 1 if allow_multiple_statements else 0) + + def send_cwd(self): + """sends the current working directory""" + with self.send_lock: + write_bytes(self.conn, ReplBackend._CHWD) + write_string(self.conn, os.getcwd()) + + def send_error(self): + """reports that an error occured to the interactive window""" + with self.send_lock: + write_bytes(self.conn, ReplBackend._ERRE) + + def send_exit(self): + """reports the that the REPL process has exited to the interactive window""" + with self.send_lock: + write_bytes(self.conn, ReplBackend._EXIT) + + def send_command_executed(self): + with self.send_lock: + write_bytes(self.conn, ReplBackend._DONE) + + def send_modules_changed(self): + with self.send_lock: + write_bytes(self.conn, ReplBackend._MODC) + + def read_line(self): + """reads a line of input from standard input""" + with self.send_lock: + write_bytes(self.conn, ReplBackend._RDLN) + self.input_event.acquire() + return self.input_string + + def write_stdout(self, value): + """writes a string to standard output in the remote console""" + with self.send_lock: + write_bytes(self.conn, ReplBackend._STDO) + write_string(self.conn, value) + + def write_stderr(self, value): + """writes a string to standard input in the remote console""" + with self.send_lock: + write_bytes(self.conn, ReplBackend._STDE) + write_string(self.conn, value) + + ################################################################ + + # Implementation of execution, etc... + + def execution_loop(self): + """starts processing execution requests""" + raise NotImplementedError + + def run_command(self, command): + """runs the specified command which is a string containing code""" + raise NotImplementedError + + def execute_file(self, filename, args): + """executes the given filename as the main module""" + return self.execute_file_ex('script', filename, args) + + def execute_file_ex(self, filetype, filename, args): + """executes the given filename as a 'script', 'module' or 'process'.""" + raise NotImplementedError + + def interrupt_main(self): + """aborts the current running command""" + raise NotImplementedError + + def exit_process(self): + """exits the REPL process""" + raise NotImplementedError + + def get_members(self, expression): + """returns a tuple of the type name, instance members, and type members""" + raise NotImplementedError + + def get_signatures(self, expression): + """returns doc, args, vargs, varkw, defaults.""" + raise NotImplementedError + + def set_current_module(self, module): + """sets the module which code executes against""" + raise NotImplementedError + + def set_current_thread_and_frame(self, thread_id, frame_id, frame_kind): + """sets the current thread and frame which code will execute against""" + raise NotImplementedError + + def get_module_names(self): + """returns a list of module names""" + raise NotImplementedError + + def flush(self): + """flushes the stdout/stderr buffers""" + raise NotImplementedError + + def attach_process(self, port, debugger_id, debug_options): + """starts processing execution requests""" + raise NotImplementedError + +def exit_work_item(): + sys.exit(0)