Skip to content

Commit ab02882

Browse files
alexdimaalexdima
committed
Extract TokensStore
1 parent cb58cf9 commit ab02882

3 files changed

Lines changed: 128 additions & 79 deletions

File tree

‎src/vs/editor/common/model/textModel.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1784,12 +1784,12 @@ export class TextModel extends Disposable implements model.ITextModel {
17841784
startLineNumber = Math.max(1, startLineNumber);
17851785
endLineNumber = Math.min(this.getLineCount(), endLineNumber);
17861786

1787-
if (endLineNumber <= this._tokens.inValidLineStartIndex) {
1787+
if (endLineNumber <= this._tokens.invalidLineStartIndex) {
17881788
// nothing to do
17891789
return;
17901790
}
17911791

1792-
if (startLineNumber <= this._tokens.inValidLineStartIndex) {
1792+
if (startLineNumber <= this._tokens.invalidLineStartIndex) {
17931793
// tokenization has reached the viewport start...
17941794
this.forceTokenization(endLineNumber);
17951795
return;
@@ -1838,7 +1838,7 @@ export class TextModel extends Disposable implements model.ITextModel {
18381838

18391839
// We cannot trust these states/tokens to be valid!
18401840
// (see https://github.com/Microsoft/vscode/issues/67607)
1841-
this._tokens._setIsInvalid(i - 1, true);
1841+
this._tokens._invalidateLine(i - 1);
18421842
this._tokens._setState(i - 1, state);
18431843
state = r.endState.clone();
18441844
eventBuilder.registerChangedTokens(i);
@@ -1884,7 +1884,7 @@ export class TextModel extends Disposable implements model.ITextModel {
18841884
return false;
18851885
}
18861886

1887-
if (lineNumber < this._tokens.inValidLineStartIndex + 1) {
1887+
if (lineNumber < this._tokens.invalidLineStartIndex + 1) {
18881888
return true;
18891889
}
18901890

‎src/vs/editor/common/model/textModelTokens.ts‎

Lines changed: 123 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -165,37 +165,22 @@ class ModelLineTokens {
165165
}
166166
}
167167

168-
export class ModelLinesTokens {
169-
170-
public readonly languageIdentifier: LanguageIdentifier;
171-
public readonly tokenizationSupport: ITokenizationSupport | null;
168+
class TokensStore {
172169
private _tokens: ModelLineTokens[];
173-
private _invalidLineStartIndex: number;
174-
private _lastState: IState | null;
170+
_invalidLineStartIndex: number;
171+
_lastState: IState | null;
175172

176-
constructor(languageIdentifier: LanguageIdentifier, tokenizationSupport: ITokenizationSupport | null) {
177-
this.languageIdentifier = languageIdentifier;
178-
this.tokenizationSupport = tokenizationSupport;
173+
constructor(initialState: IState | null) {
179174
this._tokens = [];
180-
if (this.tokenizationSupport) {
181-
let initialState: IState | null = null;
182-
try {
183-
initialState = this.tokenizationSupport.getInitialState();
184-
} catch (e) {
185-
onUnexpectedError(e);
186-
this.tokenizationSupport = null;
187-
}
188-
189-
if (initialState) {
190-
this._tokens[0] = new ModelLineTokens(initialState);
191-
}
192-
}
193-
194175
this._invalidLineStartIndex = 0;
195176
this._lastState = null;
177+
178+
if (initialState) {
179+
this._tokens[0] = new ModelLineTokens(initialState);
180+
}
196181
}
197182

198-
public get inValidLineStartIndex() {
183+
public get invalidLineStartIndex() {
199184
return this._invalidLineStartIndex;
200185
}
201186

@@ -215,15 +200,6 @@ export class ModelLinesTokens {
215200
return new LineTokens(lineTokens, lineText);
216201
}
217202

218-
public isCheapToTokenize(lineNumber: number): boolean {
219-
const firstInvalidLineNumber = this._invalidLineStartIndex + 1;
220-
return (firstInvalidLineNumber >= lineNumber);
221-
}
222-
223-
public hasLinesToTokenize(buffer: ITextBuffer): boolean {
224-
return (this._invalidLineStartIndex < buffer.getLineCount());
225-
}
226-
227203
public invalidateLine(lineIndex: number): void {
228204
this._setIsInvalid(lineIndex, true);
229205
if (lineIndex < this._invalidLineStartIndex) {
@@ -232,27 +208,27 @@ export class ModelLinesTokens {
232208
}
233209
}
234210

235-
_setIsInvalid(lineIndex: number, invalid: boolean): void {
211+
private _setIsInvalid(lineIndex: number, invalid: boolean): void {
236212
if (lineIndex < this._tokens.length && this._tokens[lineIndex]) {
237213
this._tokens[lineIndex]._invalid = invalid;
238214
}
239215
}
240216

241-
_isInvalid(lineIndex: number): boolean {
217+
public isInvalid(lineIndex: number): boolean {
242218
if (lineIndex < this._tokens.length && this._tokens[lineIndex]) {
243219
return this._tokens[lineIndex]._invalid;
244220
}
245221
return true;
246222
}
247223

248-
_getState(lineIndex: number): IState | null {
224+
public getState(lineIndex: number): IState | null {
249225
if (lineIndex < this._tokens.length && this._tokens[lineIndex]) {
250226
return this._tokens[lineIndex]._state;
251227
}
252228
return null;
253229
}
254230

255-
_setTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineTextLength: number, tokens: Uint32Array): void {
231+
public setTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineTextLength: number, tokens: Uint32Array): void {
256232
let target: ModelLineTokens;
257233
if (lineIndex < this._tokens.length && this._tokens[lineIndex]) {
258234
target = this._tokens[lineIndex];
@@ -284,7 +260,43 @@ export class ModelLinesTokens {
284260
target._lineTokens = tokens.buffer;
285261
}
286262

287-
_setState(lineIndex: number, state: IState): void {
263+
public setGoodTokens(topLevelLanguageId: LanguageId, linesLength: number, lineIndex: number, text: string, r: TokenizationResult2): void {
264+
const endStateIndex = lineIndex + 1;
265+
this.setTokens(topLevelLanguageId, lineIndex, text.length, r.tokens);
266+
this._setIsInvalid(lineIndex, false);
267+
268+
if (endStateIndex < linesLength) {
269+
const previousEndState = this.getState(endStateIndex);
270+
if (previousEndState !== null && r.endState.equals(previousEndState)) {
271+
// The end state of this line remains the same
272+
let nextInvalidLineIndex = lineIndex + 1;
273+
while (nextInvalidLineIndex < linesLength) {
274+
if (this.isInvalid(nextInvalidLineIndex)) {
275+
break;
276+
}
277+
if (nextInvalidLineIndex + 1 < linesLength) {
278+
if (this.getState(nextInvalidLineIndex + 1) === null) {
279+
break;
280+
}
281+
} else {
282+
if (this._lastState === null) {
283+
break;
284+
}
285+
}
286+
nextInvalidLineIndex++;
287+
}
288+
this._invalidLineStartIndex = nextInvalidLineIndex;
289+
} else {
290+
this._invalidLineStartIndex = lineIndex + 1;
291+
this.setState(endStateIndex, r.endState);
292+
}
293+
} else {
294+
this._lastState = r.endState;
295+
this._invalidLineStartIndex = linesLength;
296+
}
297+
}
298+
299+
public setState(lineIndex: number, state: IState): void {
288300
if (lineIndex < this._tokens.length && this._tokens[lineIndex]) {
289301
this._tokens[lineIndex]._state = state;
290302
} else {
@@ -374,14 +386,82 @@ export class ModelLinesTokens {
374386
}
375387

376388
//#endregion
389+
}
390+
391+
export class ModelLinesTokens {
392+
393+
public readonly languageIdentifier: LanguageIdentifier;
394+
public readonly tokenizationSupport: ITokenizationSupport | null;
395+
public readonly store: TokensStore;
396+
397+
constructor(languageIdentifier: LanguageIdentifier, tokenizationSupport: ITokenizationSupport | null) {
398+
this.languageIdentifier = languageIdentifier;
399+
this.tokenizationSupport = tokenizationSupport;
400+
401+
let initialState: IState | null = null;
402+
if (this.tokenizationSupport) {
403+
try {
404+
initialState = this.tokenizationSupport.getInitialState();
405+
} catch (e) {
406+
onUnexpectedError(e);
407+
this.tokenizationSupport = null;
408+
}
409+
}
410+
this.store = new TokensStore(initialState);
411+
}
412+
413+
public get invalidLineStartIndex() {
414+
return this.store.invalidLineStartIndex;
415+
}
416+
417+
public getTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineText: string): LineTokens {
418+
return this.store.getTokens(topLevelLanguageId, lineIndex, lineText);
419+
}
420+
421+
public isCheapToTokenize(lineNumber: number): boolean {
422+
const firstInvalidLineNumber = this.store.invalidLineStartIndex + 1;
423+
return (firstInvalidLineNumber >= lineNumber);
424+
}
425+
426+
public hasLinesToTokenize(buffer: ITextBuffer): boolean {
427+
return (this.store.invalidLineStartIndex < buffer.getLineCount());
428+
}
429+
430+
_isInvalid(lineIndex: number): boolean {
431+
return this.store.isInvalid(lineIndex);
432+
}
433+
434+
_getState(lineIndex: number): IState | null {
435+
return this.store.getState(lineIndex);
436+
}
437+
438+
_setTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineTextLength: number, tokens: Uint32Array): void {
439+
this.store.setTokens(topLevelLanguageId, lineIndex, lineTextLength, tokens);
440+
}
441+
442+
_setState(lineIndex: number, state: IState): void {
443+
this.store.setState(lineIndex, state);
444+
}
445+
446+
//#region Editing
447+
448+
public applyEdits(range: Range, eolCount: number, firstLineLength: number): void {
449+
this.store.applyEdits(range, eolCount, firstLineLength);
450+
}
451+
452+
_invalidateLine(lineIndex: number): void {
453+
this.store.invalidateLine(lineIndex);
454+
}
455+
456+
//#endregion
377457

378458
//#region Tokenization
379459

380460
public _tokenizeOneLine(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder): number {
381461
if (!this.hasLinesToTokenize(buffer)) {
382462
return buffer.getLineCount() + 1;
383463
}
384-
const lineNumber = this._invalidLineStartIndex + 1;
464+
const lineNumber = this.store.invalidLineStartIndex + 1;
385465
this._updateTokensUntilLine(buffer, eventBuilder, lineNumber);
386466
return lineNumber;
387467
}
@@ -405,16 +485,15 @@ export class ModelLinesTokens {
405485

406486
public _updateTokensUntilLine(buffer: ITextBuffer, eventBuilder: ModelTokensChangedEventBuilder, lineNumber: number): void {
407487
if (!this.tokenizationSupport) {
408-
this._invalidLineStartIndex = buffer.getLineCount();
488+
this.store._invalidLineStartIndex = buffer.getLineCount();
409489
return;
410490
}
411491

412492
const linesLength = buffer.getLineCount();
413493
const endLineIndex = lineNumber - 1;
414494

415495
// Validate all states up to and including endLineIndex
416-
for (let lineIndex = this._invalidLineStartIndex; lineIndex <= endLineIndex; lineIndex++) {
417-
const endStateIndex = lineIndex + 1;
496+
for (let lineIndex = this.store.invalidLineStartIndex; lineIndex <= endLineIndex; lineIndex++) {
418497
const text = buffer.getLineContent(lineIndex + 1);
419498
const lineStartState = this._getState(lineIndex);
420499

@@ -431,40 +510,10 @@ export class ModelLinesTokens {
431510
if (!r) {
432511
r = nullTokenize2(this.languageIdentifier.id, text, lineStartState, 0);
433512
}
434-
this._setTokens(this.languageIdentifier.id, lineIndex, text.length, r.tokens);
513+
this.store.setGoodTokens(this.languageIdentifier.id, linesLength, lineIndex, text, r);
435514
eventBuilder.registerChangedTokens(lineIndex + 1);
436-
this._setIsInvalid(lineIndex, false);
437-
438-
if (endStateIndex < linesLength) {
439-
const previousEndState = this._getState(endStateIndex);
440-
if (previousEndState !== null && r.endState.equals(previousEndState)) {
441-
// The end state of this line remains the same
442-
let nextInvalidLineIndex = lineIndex + 1;
443-
while (nextInvalidLineIndex < linesLength) {
444-
if (this._isInvalid(nextInvalidLineIndex)) {
445-
break;
446-
}
447-
if (nextInvalidLineIndex + 1 < linesLength) {
448-
if (this._getState(nextInvalidLineIndex + 1) === null) {
449-
break;
450-
}
451-
} else {
452-
if (this._lastState === null) {
453-
break;
454-
}
455-
}
456-
nextInvalidLineIndex++;
457-
}
458-
this._invalidLineStartIndex = Math.max(this._invalidLineStartIndex, nextInvalidLineIndex);
459-
lineIndex = nextInvalidLineIndex - 1; // -1 because the outer loop increments it
460-
} else {
461-
this._setState(endStateIndex, r.endState);
462-
}
463-
} else {
464-
this._lastState = r.endState;
465-
}
515+
lineIndex = this.store.invalidLineStartIndex - 1; // -1 because the outer loop increments it
466516
}
467-
this._invalidLineStartIndex = Math.max(this._invalidLineStartIndex, endLineIndex + 1);
468517
}
469518

470519
// #endregion

‎src/vs/editor/test/common/model/model.modes.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -198,7 +198,7 @@ suite('Editor Model - Model Modes 2', () => {
198198
for (i = 0; i < len; i++) {
199199
stateEqual(model._tokens._getState(i)!, states[i]);
200200
}
201-
stateEqual((<any>model)._tokens._lastState, states[len]);
201+
stateEqual(model._tokens.store._lastState!, states[len]);
202202
}
203203

204204
let thisModel: TextModel;

0 commit comments

Comments
 (0)