forked from DonJayamanne/pythonVSCode
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtokenizer.ts
More file actions
137 lines (119 loc) · 4.26 KB
/
tokenizer.ts
File metadata and controls
137 lines (119 loc) · 4.26 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
'use strict';
// tslint:disable-next-line:import-name
import Char from 'typescript-char';
import { CharacterStream } from './characterStream';
import { TextRangeCollection } from './textRangeCollection';
import { ICharacterStream, ITextRangeCollection, IToken, ITokenizer, TextRange, TokenType } from './types';
enum QuoteType {
None,
Single,
Double,
TripleSingle,
TripleDouble
}
class Token extends TextRange implements IToken {
public readonly type: TokenType;
constructor(type: TokenType, start: number, length: number) {
super(start, length);
this.type = type;
}
}
export class Tokenizer implements ITokenizer {
private cs: ICharacterStream;
private tokens: IToken[] = [];
public Tokenize(text: string): ITextRangeCollection<IToken>;
public Tokenize(text: string, start: number, length: number): ITextRangeCollection<IToken>;
public Tokenize(text: string, start?: number, length?: number): ITextRangeCollection<IToken> {
if (start === undefined) {
start = 0;
} else if (start < 0 || start >= text.length) {
throw new Error('Invalid range start');
}
if (length === undefined) {
length = text.length;
} else if (length < 0 || start + length >= text.length) {
throw new Error('Invalid range length');
}
this.cs = new CharacterStream(text);
this.cs.position = start;
const end = start + length;
while (!this.cs.isEndOfStream()) {
this.AddNextToken();
if (this.cs.position >= end) {
break;
}
}
return new TextRangeCollection(this.tokens);
}
private AddNextToken(): void {
this.cs.skipWhitespace();
if (this.cs.isEndOfStream()) {
return;
}
if (!this.handleCharacter()) {
this.cs.moveNext();
}
}
private handleCharacter(): boolean {
const quoteType = this.getQuoteType();
if (quoteType !== QuoteType.None) {
this.handleString(quoteType);
return true;
}
switch (this.cs.currentChar) {
case Char.Hash:
this.handleComment();
break;
default:
break;
}
return false;
}
private handleComment(): void {
const start = this.cs.position;
this.cs.skipToEol();
this.tokens.push(new Token(TokenType.Comment, start, this.cs.position - start));
}
private getQuoteType(): QuoteType {
if (this.cs.currentChar === Char.SingleQuote) {
return this.cs.nextChar === Char.SingleQuote && this.cs.lookAhead(2) === Char.SingleQuote
? QuoteType.TripleSingle
: QuoteType.Single;
}
if (this.cs.currentChar === Char.DoubleQuote) {
return this.cs.nextChar === Char.DoubleQuote && this.cs.lookAhead(2) === Char.DoubleQuote
? QuoteType.TripleDouble
: QuoteType.Double;
}
return QuoteType.None;
}
private handleString(quoteType: QuoteType): void {
const start = this.cs.position;
if (quoteType === QuoteType.Single || quoteType === QuoteType.Double) {
this.cs.moveNext();
this.skipToSingleEndQuote(quoteType === QuoteType.Single
? Char.SingleQuote
: Char.DoubleQuote);
} else {
this.cs.advance(3);
this.skipToTripleEndQuote(quoteType === QuoteType.TripleSingle
? Char.SingleQuote
: Char.DoubleQuote);
}
this.tokens.push(new Token(TokenType.String, start, this.cs.position - start));
}
private skipToSingleEndQuote(quote: number): void {
while (!this.cs.isEndOfStream() && this.cs.currentChar !== quote) {
this.cs.moveNext();
}
this.cs.moveNext();
}
private skipToTripleEndQuote(quote: number): void {
while (!this.cs.isEndOfStream() && (this.cs.currentChar !== quote || this.cs.nextChar !== quote || this.cs.lookAhead(2) !== quote)) {
this.cs.moveNext();
}
this.cs.advance(3);
}
}