Skip to content

Commit 091679e

Browse files
georgewfrasergeorgewfraser
committed
Inline tree-sitter library
1 parent a54a53a commit 091679e

5 files changed

Lines changed: 299 additions & 97 deletions

File tree

‎lib/extension.ts‎

Lines changed: 180 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -3,21 +3,184 @@
33
// Import the module and reference it with the alias vscode in your code below
44
import * as Path from "path";
55
import * as FS from "fs";
6-
import {window, workspace, ExtensionContext, commands, tasks, Task, TaskExecution, ShellExecution, Uri, TaskDefinition, languages, IndentAction, Progress, ProgressLocation, ConfigurationChangeEvent, TextDocumentChangeEvent, Hover, debug, DebugConfiguration, Range, Position, TextDocument} from 'vscode';
6+
import {window, workspace, ExtensionContext, commands, tasks, Task, TaskExecution, ShellExecution, Uri, TaskDefinition, languages, IndentAction, Progress, ProgressLocation, ConfigurationChangeEvent, TextDocumentChangeEvent, debug, DebugConfiguration, Range, Position, TextDocument, TextEditor, TextEditorDecorationType, DecorationRangeBehavior, DecorationRenderOptions} from 'vscode';
77
import {LanguageClient, LanguageClientOptions, ServerOptions, NotificationType} from "vscode-languageclient";
8-
import {tree, activate as activateTreeSitter, decoration} from 'vscode-tree-sitter';
9-
import {color} from './treeSitter';
108
import * as path from 'path';
9+
import * as scopes from './scopes'
10+
import Parser = require("web-tree-sitter");
11+
import { colorJava, Range as ColorRange } from "./treeSitterColor";
1112

1213
// If we want to profile using VisualVM, we have to run the language server using regular java, not jlink
1314
const visualVm = false;
1415

15-
// Show tree-sitter parse tree when you hover over code
16-
const debugTreeSitter = false;
16+
// Create decoration types from scopes lazily
17+
const decorationCache = new Map<string, TextEditorDecorationType>()
18+
function decoration(scope: string): TextEditorDecorationType|undefined {
19+
// If we've already created a decoration for `scope`, use it
20+
if (decorationCache.has(scope)) {
21+
return decorationCache.get(scope)
22+
}
23+
// If `scope` is defined in the current theme, create a decoration for it
24+
const textmate = scopes.find(scope)
25+
if (textmate) {
26+
const decoration = createDecorationFromTextmate(textmate)
27+
decorationCache.set(scope, decoration)
28+
return decoration
29+
}
30+
// Otherwise, give up, there is no color available for this scope
31+
return undefined
32+
}
33+
function createDecorationFromTextmate(themeStyle: scopes.TextMateRuleSettings): TextEditorDecorationType {
34+
let options: DecorationRenderOptions = {}
35+
options.rangeBehavior = DecorationRangeBehavior.OpenOpen
36+
if (themeStyle.foreground) {
37+
options.color = themeStyle.foreground
38+
}
39+
if (themeStyle.background) {
40+
options.backgroundColor = themeStyle.background
41+
}
42+
if (themeStyle.fontStyle) {
43+
let parts: string[] = themeStyle.fontStyle.split(" ")
44+
parts.forEach((part) => {
45+
switch (part) {
46+
case "italic":
47+
options.fontStyle = "italic"
48+
break
49+
case "bold":
50+
options.fontWeight = "bold"
51+
break
52+
case "underline":
53+
options.textDecoration = "underline"
54+
break
55+
default:
56+
break
57+
}
58+
})
59+
}
60+
return window.createTextEditorDecorationType(options)
61+
}
62+
63+
// Load styles from the current active theme
64+
async function loadStyles() {
65+
await scopes.load()
66+
// Clear old styles
67+
for (const style of decorationCache.values()) {
68+
style.dispose()
69+
}
70+
decorationCache.clear()
71+
}
72+
73+
// For some reason this crashes if we put it inside activate
74+
const initParser = Parser.init() // TODO this isn't a field, suppress package member coloring like Go
75+
76+
var javaParser: Parser = null
1777

1878
/** Called when extension is activated */
1979
export function activate(context: ExtensionContext) {
2080
console.log('Activating Java');
81+
82+
// Parse of all visible documents
83+
const trees: {[uri: string]: Parser.Tree} = {}
84+
async function open(editor: TextEditor) {
85+
if (editor.document.languageId != 'java') return
86+
if (javaParser == null) {
87+
const absolute = path.join(context.extensionPath, 'lib', 'tree-sitter-java.wasm');
88+
const wasm = path.relative(process.cwd(), absolute)
89+
const lang = await Parser.Language.load(wasm)
90+
javaParser = new Parser()
91+
javaParser.setLanguage(lang)
92+
}
93+
const t = javaParser.parse(editor.document.getText()) // TODO don't use getText, use Parser.Input
94+
trees[editor.document.uri.toString()] = t
95+
colorUri(editor.document.uri)
96+
}
97+
// NOTE: if you make this an async function, it seems to cause edit anomalies
98+
function edit(edit: TextDocumentChangeEvent) {
99+
if (edit.document.languageId != 'java') return
100+
updateTree(javaParser, edit)
101+
colorUri(edit.document.uri)
102+
}
103+
function updateTree(parser: Parser, edit: TextDocumentChangeEvent) {
104+
if (edit.contentChanges.length == 0) return
105+
const old = trees[edit.document.uri.toString()]
106+
for (const e of edit.contentChanges) {
107+
const startIndex = e.rangeOffset
108+
const oldEndIndex = e.rangeOffset + e.rangeLength
109+
const newEndIndex = e.rangeOffset + e.text.length
110+
const startPos = edit.document.positionAt(startIndex)
111+
const oldEndPos = edit.document.positionAt(oldEndIndex)
112+
const newEndPos = edit.document.positionAt(newEndIndex)
113+
const startPosition = asPoint(startPos)
114+
const oldEndPosition = asPoint(oldEndPos)
115+
const newEndPosition = asPoint(newEndPos)
116+
const delta = {startIndex, oldEndIndex, newEndIndex, startPosition, oldEndPosition, newEndPosition}
117+
old.edit(delta)
118+
}
119+
const t = parser.parse(edit.document.getText(), old) // TODO don't use getText, use Parser.Input
120+
trees[edit.document.uri.toString()] = t
121+
}
122+
function asPoint(pos: Position): Parser.Point {
123+
return {row: pos.line, column: pos.character}
124+
}
125+
function close(doc: TextDocument) {
126+
delete trees[doc.uri.toString()]
127+
}
128+
function colorUri(uri: Uri) {
129+
for (const editor of window.visibleTextEditors) {
130+
if (editor.document.uri == uri) {
131+
colorEditor(editor)
132+
}
133+
}
134+
}
135+
const warnedScopes = new Set<string>()
136+
function colorEditor(editor: TextEditor) {
137+
const t = trees[editor.document.uri.toString()]
138+
if (t == null) return
139+
if (editor.document.languageId != 'java') return;
140+
const scopes = colorJava(t.rootNode, visibleLines(editor))
141+
for (const scope of scopes.keys()) {
142+
const dec = decoration(scope)
143+
if (dec) {
144+
const ranges = scopes.get(scope)!.map(range)
145+
editor.setDecorations(dec, ranges)
146+
} else if (!warnedScopes.has(scope)) {
147+
console.warn(scope, 'was not found in the current theme')
148+
warnedScopes.add(scope)
149+
}
150+
}
151+
for (const scope of decorationCache.keys()) {
152+
if (!scopes.has(scope)) {
153+
const dec = decorationCache.get(scope)!
154+
editor.setDecorations(dec, [])
155+
}
156+
}
157+
}
158+
async function colorAllOpen() {
159+
for (const editor of window.visibleTextEditors) {
160+
await open(editor)
161+
}
162+
}
163+
// Load active color theme
164+
async function onChangeConfiguration(event: ConfigurationChangeEvent) {
165+
let colorizationNeedsReload: boolean = event.affectsConfiguration("workbench.colorTheme")
166+
|| event.affectsConfiguration("editor.tokenColorCustomizations")
167+
if (colorizationNeedsReload) {
168+
await loadStyles()
169+
colorAllOpen()
170+
}
171+
}
172+
context.subscriptions.push(workspace.onDidChangeConfiguration(onChangeConfiguration))
173+
context.subscriptions.push(window.onDidChangeVisibleTextEditors(colorAllOpen))
174+
context.subscriptions.push(workspace.onDidChangeTextDocument(edit))
175+
context.subscriptions.push(workspace.onDidCloseTextDocument(close))
176+
context.subscriptions.push(window.onDidChangeTextEditorVisibleRanges(change => colorEditor(change.textEditor)))
177+
// Don't wait for the initial color, it takes too long to inspect the themes and causes VSCode extension host to hang
178+
async function activateLazily() {
179+
await loadStyles()
180+
await initParser
181+
colorAllOpen()
182+
}
183+
activateLazily()
21184

22185
// Options to control the language client
23186
let clientOptions: LanguageClientOptions = {
@@ -108,52 +271,6 @@ export function activate(context: ExtensionContext) {
108271
context.subscriptions.push(window.onDidChangeVisibleTextEditors(applySemanticColors));
109272
workspace.onDidCloseTextDocument(forgetSemanticColors);
110273
});
111-
112-
// Parse .java files incrementally using tree-sitter
113-
const parserPath = path.join(context.extensionPath, 'lib', 'tree-sitter-java.wasm');
114-
activateTreeSitter(context, {'java': {wasm: parserPath}}).then(colorAll);
115-
function colorAll() {
116-
for (const editor of window.visibleTextEditors) {
117-
color(editor);
118-
}
119-
}
120-
function colorEdited(evt: TextDocumentChangeEvent) {
121-
for (const editor of window.visibleTextEditors) {
122-
if (editor.document.uri.toString() === evt.document.uri.toString()) {
123-
color(editor);
124-
}
125-
}
126-
}
127-
function onChangeConfiguration(event: ConfigurationChangeEvent) {
128-
const colorizationNeedsReload: boolean = event.affectsConfiguration('workbench.colorTheme')
129-
|| event.affectsConfiguration('editor.tokenColorCustomizations');
130-
if (colorizationNeedsReload) {
131-
colorAll();
132-
}
133-
}
134-
context.subscriptions.push(workspace.onDidChangeConfiguration(onChangeConfiguration));
135-
context.subscriptions.push(window.onDidChangeVisibleTextEditors(colorAll));
136-
context.subscriptions.push(window.onDidChangeTextEditorVisibleRanges(change => color(change.textEditor)));
137-
context.subscriptions.push(workspace.onDidChangeTextDocument(colorEdited));
138-
if (debugTreeSitter) {
139-
languages.registerHoverProvider({scheme: 'file', language: 'java'}, {
140-
provideHover(doc, pos) {
141-
const offset = doc.offsetAt(pos);
142-
let node = tree(doc.uri).rootNode;
143-
recur:
144-
while (true) {
145-
for (const child of node.namedChildren) {
146-
if (child.startIndex <= offset && offset <= child.endIndex) {
147-
node = child;
148-
continue recur;
149-
}
150-
}
151-
break;
152-
}
153-
return new Hover(node.toString());
154-
}
155-
});
156-
}
157274
}
158275

159276
// this method is called when your extension is deactivated
@@ -485,4 +602,16 @@ function enableJavadocSymbols() {
485602
}
486603
]
487604
});
605+
}
606+
607+
function range(x: ColorRange): Range {
608+
return new Range(x.start.row, x.start.column, x.end.row, x.end.column)
609+
}
610+
611+
function visibleLines(editor: TextEditor) {
612+
return editor.visibleRanges.map(range => {
613+
const start = range.start.line
614+
const end = range.end.line
615+
return {start, end}
616+
})
488617
}

‎lib/scopes.ts‎

Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
import * as vscode from 'vscode'
2+
import * as path from 'path'
3+
import * as fs from 'fs'
4+
import * as jsonc from "jsonc-parser"
5+
6+
export interface TextMateRule {
7+
scope: string|string[]
8+
settings: TextMateRuleSettings
9+
}
10+
11+
export interface TextMateRuleSettings {
12+
foreground: string | undefined
13+
background: string | undefined
14+
fontStyle: string | undefined
15+
}
16+
17+
// Current theme colors
18+
const colors = new Map<string, TextMateRuleSettings>()
19+
20+
export function find(scope: string): TextMateRuleSettings|undefined {
21+
return colors.get(scope)
22+
}
23+
24+
// Load all textmate scopes in the currently active theme
25+
export async function load() {
26+
// Remove any previous theme
27+
colors.clear()
28+
// Find out current color theme
29+
const themeName = vscode.workspace.getConfiguration("workbench").get("colorTheme")
30+
if (typeof themeName != 'string') {
31+
console.warn('workbench.colorTheme is', themeName)
32+
return
33+
}
34+
// Try to load colors from that theme
35+
try {
36+
await loadThemeNamed(themeName)
37+
} catch(e) {
38+
console.warn('failed to load theme', themeName, e)
39+
}
40+
}
41+
42+
// Find current theme on disk
43+
async function loadThemeNamed(themeName: string) {
44+
for (const extension of vscode.extensions.all) {
45+
const extensionPath: string = extension.extensionPath
46+
const extensionPackageJsonPath: string = path.join(extensionPath, "package.json")
47+
if (!await checkFileExists(extensionPackageJsonPath)) {
48+
continue
49+
}
50+
const packageJsonText: string = await readFileText(extensionPackageJsonPath)
51+
const packageJson: any = jsonc.parse(packageJsonText)
52+
if (packageJson.contributes && packageJson.contributes.themes) {
53+
for (const theme of packageJson.contributes.themes) {
54+
const id = theme.id || theme.label
55+
if (id == themeName) {
56+
const themeRelativePath: string = theme.path
57+
const themeFullPath: string = path.join(extensionPath, themeRelativePath)
58+
await loadThemeFile(themeFullPath)
59+
}
60+
}
61+
}
62+
}
63+
}
64+
65+
async function loadThemeFile(themePath: string) {
66+
if (await checkFileExists(themePath)) {
67+
const themeContentText: string = await readFileText(themePath)
68+
const themeContent: any = jsonc.parse(themeContentText)
69+
if (themeContent && themeContent.tokenColors) {
70+
loadColors(themeContent.tokenColors)
71+
if (themeContent.include) {
72+
// parse included theme file
73+
const includedThemePath: string = path.join(path.dirname(themePath), themeContent.include)
74+
await loadThemeFile(includedThemePath)
75+
}
76+
}
77+
}
78+
}
79+
80+
function loadColors(textMateRules: TextMateRule[]): void {
81+
for (const rule of textMateRules) {
82+
if (typeof rule.scope == 'string') {
83+
if (!colors.has(rule.scope)) {
84+
colors.set(rule.scope, rule.settings)
85+
}
86+
} else if (rule.scope instanceof Array) {
87+
for (const scope of rule.scope) {
88+
if (!colors.has(scope)) {
89+
colors.set(scope, rule.settings)
90+
}
91+
}
92+
}
93+
}
94+
}
95+
96+
function checkFileExists(filePath: string): Promise<boolean> {
97+
return new Promise((resolve, _) => {
98+
fs.stat(filePath, (_, stats) => {
99+
if (stats && stats.isFile()) {
100+
resolve(true)
101+
} else {
102+
console.warn('no such file', filePath)
103+
resolve(false)
104+
}
105+
})
106+
})
107+
}
108+
109+
function readFileText(filePath: string, encoding: string = "utf8"): Promise<string> {
110+
return new Promise<string>((resolve, reject) => {
111+
fs.readFile(filePath, encoding, (err, data) => {
112+
if (err) {
113+
reject(err)
114+
} else {
115+
resolve(data)
116+
}
117+
})
118+
})
119+
}

0 commit comments

Comments
 (0)