forked from DonJayamanne/pythonVSCode
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtokenizer.ts
More file actions
96 lines (88 loc) · 3.17 KB
/
tokenizer.ts
File metadata and controls
96 lines (88 loc) · 3.17 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
'use strict';
import { wireTmGrammars } from 'monaco-editor-textmate';
import * as monacoEditor from 'monaco-editor/esm/vs/editor/editor.api';
import { Registry } from 'monaco-textmate';
import { loadWASM } from 'onigasm';
import { PYTHON_LANGUAGE } from '../../client/common/constants';
export function registerMonacoLanguage() {
// Tell monaco about our language
monacoEditor.languages.register({
id: PYTHON_LANGUAGE,
extensions: ['.py']
});
// Setup the configuration so that auto indent and other things work. Onigasm is just going to setup the tokenizer
monacoEditor.languages.setLanguageConfiguration(
PYTHON_LANGUAGE,
{
comments: {
lineComment: '#',
blockComment: ['\'\'\'', '\"\"\"']
},
brackets: [
['{', '}'],
['[', ']'],
['(', ')']
],
autoClosingPairs: [
{ open: '{', close: '}' },
{ open: '[', close: ']' },
{ open: '(', close: ')' },
{ open: '"', close: '"', notIn: ['string'] },
{ open: '\'', close: '\'', notIn: ['string', 'comment'] }
],
surroundingPairs: [
{ open: '{', close: '}' },
{ open: '[', close: ']' },
{ open: '(', close: ')' },
{ open: '"', close: '"' },
{ open: '\'', close: '\'' }
],
onEnterRules: [
{
beforeText: new RegExp('^\\s*(?:def|class|for|if|elif|else|while|try|with|finally|except|async).*?:\\s*$'),
action: { indentAction: monacoEditor.languages.IndentAction.Indent }
}
],
folding: {
offSide: true,
markers: {
start: new RegExp('^\\s*#region\\b'),
end: new RegExp('^\\s*#endregion\\b')
}
}
}
);
}
// tslint:disable: no-any
export async function initializeTokenizer(
getOnigasm: () => Promise<ArrayBuffer>,
getTmlanguageJSON: () => Promise<string>,
loadingFinished: (e?: any) => void): Promise<void> {
try {
// Register the language first
registerMonacoLanguage();
// Load the web assembly
const blob = await getOnigasm();
await loadWASM(blob);
// Setup our registry of different
const registry = new Registry({
getGrammarDefinition: async (_scopeName) => {
return {
format: 'json',
content: await getTmlanguageJSON()
};
}
});
// map of monaco "language id's" to TextMate scopeNames
const grammars = new Map();
grammars.set('python', 'source.python');
// Wire everything together.
await wireTmGrammars(monacoEditor, registry, grammars);
// Indicate to the callback that we're done.
loadingFinished();
} catch (e) {
loadingFinished(e);
}
}