Merge pull request microsoft#51265 from wistcc/WC/51127

alexdima · web-flow · commit b61725b1e8cb · 2018-06-11T12:52:29.000+02:00
Adding warning when tokenization is skipped
diff --git a/src/vs/workbench/services/textMate/electron-browser/TMSyntax.ts b/src/vs/workbench/services/textMate/electron-browser/TMSyntax.ts
@@ -22,6 +22,7 @@ import { TokenizationResult, TokenizationResult2 } from 'vs/editor/common/core/t
 import { nullTokenize2 } from 'vs/editor/common/modes/nullMode';
 import { generateTokensCSSForColorMap } from 'vs/editor/common/modes/supports/tokenization';
 import { Color } from 'vs/base/common/color';
+import { INotificationService } from 'vs/platform/notification/common/notification';
 import URI from 'vs/base/common/uri';
 
 export class TMScopeRegistry {
@@ -139,6 +140,7 @@ export class TextMateService implements ITextMateService {
 	private _scopeRegistry: TMScopeRegistry;
 	private _injections: { [scopeName: string]: string[]; };
 	private _injectedEmbeddedLanguages: { [scopeName: string]: IEmbeddedLanguagesMap[]; };
+	private _notificationService: INotificationService;
 
 	private _languageToScope: Map<string, string>;
 	private _styleElement: HTMLStyleElement;
@@ -149,7 +151,8 @@ export class TextMateService implements ITextMateService {
 
 	constructor(
 		@IModeService modeService: IModeService,
-		@IWorkbenchThemeService themeService: IWorkbenchThemeService
+		@IWorkbenchThemeService themeService: IWorkbenchThemeService,
+		@INotificationService notificationService: INotificationService
 	) {
 		this._styleElement = dom.createStyleSheet();
 		this._styleElement.className = 'vscode-tokens-styles';
@@ -160,6 +163,7 @@ export class TextMateService implements ITextMateService {
 		this._injections = {};
 		this._injectedEmbeddedLanguages = {};
 		this._languageToScope = new Map<string, string>();
+		this._notificationService = notificationService;
 
 		this._grammarRegistry = null;
 
@@ -383,7 +387,7 @@ export class TextMateService implements ITextMateService {
 
 	private registerDefinition(modeId: string): void {
 		this._createGrammar(modeId).then((r) => {
-			TokenizationRegistry.register(modeId, new TMTokenization(this._scopeRegistry, r.languageId, r.grammar, r.initialState, r.containsEmbeddedLanguages));
+			TokenizationRegistry.register(modeId, new TMTokenization(this._scopeRegistry, r.languageId, r.grammar, r.initialState, r.containsEmbeddedLanguages, this._notificationService));
 		}, onUnexpectedError);
 	}
 }
@@ -396,8 +400,9 @@ class TMTokenization implements ITokenizationSupport {
 	private readonly _containsEmbeddedLanguages: boolean;
 	private readonly _seenLanguages: boolean[];
 	private readonly _initialState: StackElement;
+	private _tokenizationWarningAlreadyShown: boolean;
 
-	constructor(scopeRegistry: TMScopeRegistry, languageId: LanguageId, grammar: IGrammar, initialState: StackElement, containsEmbeddedLanguages: boolean) {
+	constructor(scopeRegistry: TMScopeRegistry, languageId: LanguageId, grammar: IGrammar, initialState: StackElement, containsEmbeddedLanguages: boolean, @INotificationService private notificationService: INotificationService) {
 		this._scopeRegistry = scopeRegistry;
 		this._languageId = languageId;
 		this._grammar = grammar;
@@ -421,6 +426,10 @@ class TMTokenization implements ITokenizationSupport {
 
 		// Do not attempt to tokenize if a line has over 20k
 		if (line.length >= 20000) {
+			if (!this._tokenizationWarningAlreadyShown) {
+				this._tokenizationWarningAlreadyShown = true;
+				this.notificationService.warn(nls.localize('too many characters', "Tokenization is skipped for lines longer than 20k characters for performance reasons."));
+			}
 			console.log(`Line (${line.substr(0, 15)}...): longer than 20k characters, tokenization skipped.`);
 			return nullTokenize2(this._languageId, line, state, offsetDelta);
 		}
