Merge pull request microsoft#19976 from Microsoft/optimizeParser

ahejlsberg · web-flow · commit b6f96052d895 · 2017-11-16T10:19:10.000-08:00
Optimize parsing
diff --git a/src/compiler/binder.ts b/src/compiler/binder.ts
@@ -1835,7 +1835,7 @@ namespace ts {
         }
 
         function checkStrictModeNumericLiteral(node: NumericLiteral) {
-            if (inStrictMode && node.numericLiteralFlags & NumericLiteralFlags.Octal) {
+            if (inStrictMode && node.numericLiteralFlags & TokenFlags.Octal) {
                 file.bindDiagnostics.push(createDiagnosticForNode(node, Diagnostics.Octal_literals_are_not_allowed_in_strict_mode));
             }
         }
@@ -3319,7 +3319,7 @@ namespace ts {
                 break;
 
             case SyntaxKind.NumericLiteral:
-                if ((<NumericLiteral>node).numericLiteralFlags & NumericLiteralFlags.BinaryOrOctalSpecifier) {
+                if ((<NumericLiteral>node).numericLiteralFlags & TokenFlags.BinaryOrOctalSpecifier) {
                     transformFlags |= TransformFlags.AssertES2015;
                 }
                 break;
diff --git a/src/compiler/checker.ts b/src/compiler/checker.ts
@@ -26274,7 +26274,7 @@ namespace ts {
 
         function checkGrammarNumericLiteral(node: NumericLiteral): boolean {
             // Grammar checking
-            if (node.numericLiteralFlags & NumericLiteralFlags.Octal) {
+            if (node.numericLiteralFlags & TokenFlags.Octal) {
                 let diagnosticMessage: DiagnosticMessage | undefined;
                 if (languageVersion >= ScriptTarget.ES5) {
                     diagnosticMessage = Diagnostics.Octal_literals_are_not_available_when_targeting_ECMAScript_5_and_higher_Use_the_syntax_0;
diff --git a/src/compiler/parser.ts b/src/compiler/parser.ts
diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts
@@ -27,7 +27,7 @@ namespace ts {
         isReservedWord(): boolean;
         isUnterminated(): boolean;
         /* @internal */
-        getNumericLiteralFlags(): NumericLiteralFlags;
+        getTokenFlags(): TokenFlags;
         reScanGreaterToken(): SyntaxKind;
         reScanSlashToken(): SyntaxKind;
         reScanTemplateToken(): SyntaxKind;
@@ -815,10 +815,7 @@ namespace ts {
 
         let token: SyntaxKind;
         let tokenValue: string;
-        let precedingLineBreak: boolean;
-        let hasExtendedUnicodeEscape: boolean;
-        let tokenIsUnterminated: boolean;
-        let numericLiteralFlags: NumericLiteralFlags;
+        let tokenFlags: TokenFlags;
 
         setText(text, start, length);
 
@@ -829,12 +826,12 @@ namespace ts {
             getTokenPos: () => tokenPos,
             getTokenText: () => text.substring(tokenPos, pos),
             getTokenValue: () => tokenValue,
-            hasExtendedUnicodeEscape: () => hasExtendedUnicodeEscape,
-            hasPrecedingLineBreak: () => precedingLineBreak,
+            hasExtendedUnicodeEscape: () => (tokenFlags & TokenFlags.ExtendedUnicodeEscape) !== 0,
+            hasPrecedingLineBreak: () => (tokenFlags & TokenFlags.PrecedingLineBreak) !== 0,
             isIdentifier: () => token === SyntaxKind.Identifier || token > SyntaxKind.LastReservedWord,
             isReservedWord: () => token >= SyntaxKind.FirstReservedWord && token <= SyntaxKind.LastReservedWord,
-            isUnterminated: () => tokenIsUnterminated,
-            getNumericLiteralFlags: () => numericLiteralFlags,
+            isUnterminated: () => (tokenFlags & TokenFlags.Unterminated) !== 0,
+            getTokenFlags: () => tokenFlags,
             reScanGreaterToken,
             reScanSlashToken,
             reScanTemplateToken,
@@ -871,7 +868,7 @@ namespace ts {
             let end = pos;
             if (text.charCodeAt(pos) === CharacterCodes.E || text.charCodeAt(pos) === CharacterCodes.e) {
                 pos++;
-                numericLiteralFlags = NumericLiteralFlags.Scientific;
+                tokenFlags |= TokenFlags.Scientific;
                 if (text.charCodeAt(pos) === CharacterCodes.plus || text.charCodeAt(pos) === CharacterCodes.minus) pos++;
                 if (isDigit(text.charCodeAt(pos))) {
                     pos++;
@@ -943,7 +940,7 @@ namespace ts {
             while (true) {
                 if (pos >= end) {
                     result += text.substring(start, pos);
-                    tokenIsUnterminated = true;
+                    tokenFlags |= TokenFlags.Unterminated;
                     error(Diagnostics.Unterminated_string_literal);
                     break;
                 }
@@ -961,7 +958,7 @@ namespace ts {
                 }
                 if (isLineBreak(ch)) {
                     result += text.substring(start, pos);
-                    tokenIsUnterminated = true;
+                    tokenFlags |= TokenFlags.Unterminated;
                     error(Diagnostics.Unterminated_string_literal);
                     break;
                 }
@@ -985,7 +982,7 @@ namespace ts {
             while (true) {
                 if (pos >= end) {
                     contents += text.substring(start, pos);
-                    tokenIsUnterminated = true;
+                    tokenFlags |= TokenFlags.Unterminated;
                     error(Diagnostics.Unterminated_template_literal);
                     resultingToken = startedWithBacktick ? SyntaxKind.NoSubstitutionTemplateLiteral : SyntaxKind.TemplateTail;
                     break;
@@ -1071,7 +1068,7 @@ namespace ts {
                 case CharacterCodes.u:
                     // '\u{DDDDDDDD}'
                     if (pos < end && text.charCodeAt(pos) === CharacterCodes.openBrace) {
-                        hasExtendedUnicodeEscape = true;
+                        tokenFlags |= TokenFlags.ExtendedUnicodeEscape;
                         pos++;
                         return scanExtendedUnicodeEscape();
                     }
@@ -1240,10 +1237,7 @@ namespace ts {
 
         function scan(): SyntaxKind {
             startPos = pos;
-            hasExtendedUnicodeEscape = false;
-            precedingLineBreak = false;
-            tokenIsUnterminated = false;
-            numericLiteralFlags = 0;
+            tokenFlags = 0;
             while (true) {
                 tokenPos = pos;
                 if (pos >= end) {
@@ -1265,7 +1259,7 @@ namespace ts {
                 switch (ch) {
                     case CharacterCodes.lineFeed:
                     case CharacterCodes.carriageReturn:
-                        precedingLineBreak = true;
+                        tokenFlags |= TokenFlags.PrecedingLineBreak;
                         if (skipTrivia) {
                             pos++;
                             continue;
@@ -1396,6 +1390,9 @@ namespace ts {
                         // Multi-line comment
                         if (text.charCodeAt(pos + 1) === CharacterCodes.asterisk) {
                             pos += 2;
+                            if (text.charCodeAt(pos) === CharacterCodes.asterisk && text.charCodeAt(pos + 1) !== CharacterCodes.slash) {
+                                tokenFlags |= TokenFlags.PrecedingJSDocComment;
+                            }
 
                             let commentClosed = false;
                             while (pos < end) {
@@ -1408,7 +1405,7 @@ namespace ts {
                                 }
 
                                 if (isLineBreak(ch)) {
-                                    precedingLineBreak = true;
+                                    tokenFlags |= TokenFlags.PrecedingLineBreak;
                                 }
                                 pos++;
                             }
@@ -1421,7 +1418,9 @@ namespace ts {
                                 continue;
                             }
                             else {
-                                tokenIsUnterminated = !commentClosed;
+                                if (!commentClosed) {
+                                    tokenFlags |= TokenFlags.Unterminated;
+                                }
                                 return token = SyntaxKind.MultiLineCommentTrivia;
                             }
                         }
@@ -1442,7 +1441,7 @@ namespace ts {
                                 value = 0;
                             }
                             tokenValue = "" + value;
-                            numericLiteralFlags = NumericLiteralFlags.HexSpecifier;
+                            tokenFlags |= TokenFlags.HexSpecifier;
                             return token = SyntaxKind.NumericLiteral;
                         }
                         else if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.B || text.charCodeAt(pos + 1) === CharacterCodes.b)) {
@@ -1453,7 +1452,7 @@ namespace ts {
                                 value = 0;
                             }
                             tokenValue = "" + value;
-                            numericLiteralFlags = NumericLiteralFlags.BinarySpecifier;
+                            tokenFlags |= TokenFlags.BinarySpecifier;
                             return token = SyntaxKind.NumericLiteral;
                         }
                         else if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.O || text.charCodeAt(pos + 1) === CharacterCodes.o)) {
@@ -1464,13 +1463,13 @@ namespace ts {
                                 value = 0;
                             }
                             tokenValue = "" + value;
-                            numericLiteralFlags = NumericLiteralFlags.OctalSpecifier;
+                            tokenFlags |= TokenFlags.OctalSpecifier;
                             return token = SyntaxKind.NumericLiteral;
                         }
                         // Try to parse as an octal
                         if (pos + 1 < end && isOctalDigit(text.charCodeAt(pos + 1))) {
                             tokenValue = "" + scanOctalDigits();
-                            numericLiteralFlags = NumericLiteralFlags.Octal;
+                            tokenFlags |= TokenFlags.Octal;
                             return token = SyntaxKind.NumericLiteral;
                         }
                         // This fall-through is a deviation from the EcmaScript grammar. The grammar says that a leading zero
@@ -1627,7 +1626,7 @@ namespace ts {
                             continue;
                         }
                         else if (isLineBreak(ch)) {
-                            precedingLineBreak = true;
+                            tokenFlags |= TokenFlags.PrecedingLineBreak;
                             pos++;
                             continue;
                         }
@@ -1670,14 +1669,14 @@ namespace ts {
                     // If we reach the end of a file, or hit a newline, then this is an unterminated
                     // regex.  Report error and return what we have so far.
                     if (p >= end) {
-                        tokenIsUnterminated = true;
+                        tokenFlags |= TokenFlags.Unterminated;
                         error(Diagnostics.Unterminated_regular_expression_literal);
                         break;
                     }
 
                     const ch = text.charCodeAt(p);
                     if (isLineBreak(ch)) {
-                        tokenIsUnterminated = true;
+                        tokenFlags |= TokenFlags.Unterminated;
                         error(Diagnostics.Unterminated_regular_expression_literal);
                         break;
                     }
@@ -1895,7 +1894,7 @@ namespace ts {
             const saveTokenPos = tokenPos;
             const saveToken = token;
             const saveTokenValue = tokenValue;
-            const savePrecedingLineBreak = precedingLineBreak;
+            const saveTokenFlags = tokenFlags;
             const result = callback();
 
             // If our callback returned something 'falsy' or we're just looking ahead,
@@ -1906,7 +1905,7 @@ namespace ts {
                 tokenPos = saveTokenPos;
                 token = saveToken;
                 tokenValue = saveTokenValue;
-                precedingLineBreak = savePrecedingLineBreak;
+                tokenFlags = saveTokenFlags;
             }
             return result;
         }
@@ -1917,10 +1916,8 @@ namespace ts {
             const saveStartPos = startPos;
             const saveTokenPos = tokenPos;
             const saveToken = token;
-            const savePrecedingLineBreak = precedingLineBreak;
             const saveTokenValue = tokenValue;
-            const saveHasExtendedUnicodeEscape = hasExtendedUnicodeEscape;
-            const saveTokenIsUnterminated = tokenIsUnterminated;
+            const saveTokenFlags = tokenFlags;
 
             setText(text, start, length);
             const result = callback();
@@ -1930,10 +1927,8 @@ namespace ts {
             startPos = saveStartPos;
             tokenPos = saveTokenPos;
             token = saveToken;
-            precedingLineBreak = savePrecedingLineBreak;
             tokenValue = saveTokenValue;
-            hasExtendedUnicodeEscape = saveHasExtendedUnicodeEscape;
-            tokenIsUnterminated = saveTokenIsUnterminated;
+            tokenFlags = saveTokenFlags;
 
             return result;
         }
@@ -1974,11 +1969,8 @@ namespace ts {
             startPos = textPos;
             tokenPos = textPos;
             token = SyntaxKind.Unknown;
-            precedingLineBreak = false;
-
             tokenValue = undefined;
-            hasExtendedUnicodeEscape = false;
-            tokenIsUnterminated = false;
+            tokenFlags = 0;
         }
     }
 }
diff --git a/src/compiler/transformers/es2015.ts b/src/compiler/transformers/es2015.ts
@@ -3633,7 +3633,7 @@ namespace ts {
          * @param node A string literal.
          */
         function visitNumericLiteral(node: NumericLiteral) {
-            if (node.numericLiteralFlags & NumericLiteralFlags.BinaryOrOctalSpecifier) {
+            if (node.numericLiteralFlags & TokenFlags.BinaryOrOctalSpecifier) {
                 return setTextRange(createNumericLiteral(node.text), node);
             }
             return node;
diff --git a/src/compiler/types.ts b/src/compiler/types.ts
@@ -1474,20 +1474,25 @@ namespace ts {
     }
 
     /* @internal */
-    export const enum NumericLiteralFlags {
+    export const enum TokenFlags {
         None = 0,
-        Scientific = 1 << 1,        // e.g. `10e2`
-        Octal = 1 << 2,             // e.g. `0777`
-        HexSpecifier = 1 << 3,      // e.g. `0x00000000`
-        BinarySpecifier = 1 << 4,   // e.g. `0b0110010000000000`
-        OctalSpecifier = 1 << 5,    // e.g. `0o777`
+        PrecedingLineBreak = 1 << 0,
+        PrecedingJSDocComment = 1 << 1,
+        Unterminated = 1 << 2,
+        ExtendedUnicodeEscape = 1 << 3,
+        Scientific = 1 << 4,        // e.g. `10e2`
+        Octal = 1 << 5,             // e.g. `0777`
+        HexSpecifier = 1 << 6,      // e.g. `0x00000000`
+        BinarySpecifier = 1 << 7,   // e.g. `0b0110010000000000`
+        OctalSpecifier = 1 << 8,    // e.g. `0o777`
         BinaryOrOctalSpecifier = BinarySpecifier | OctalSpecifier,
+        NumericLiteralFlags = Scientific | Octal | HexSpecifier | BinarySpecifier | OctalSpecifier
     }
 
     export interface NumericLiteral extends LiteralExpression {
         kind: SyntaxKind.NumericLiteral;
         /* @internal */
-        numericLiteralFlags?: NumericLiteralFlags;
+        numericLiteralFlags?: TokenFlags;
     }
 
     export interface TemplateHead extends LiteralLikeNode {
diff --git a/tests/baselines/reference/duplicateIdentifierDifferentSpelling.js b/tests/baselines/reference/duplicateIdentifierDifferentSpelling.js
@@ -10,7 +10,7 @@ var X = { 0b11: '', 3: '' };
 //// [duplicateIdentifierDifferentSpelling.js]
 var A = /** @class */ (function () {
     function A() {
-        this[0b11] = '';
+        this[3] = '';
         this[3] = '';
     }
     return A;
diff --git a/tests/baselines/reference/parserArrowFunctionExpression6.errors.txt b/tests/baselines/reference/parserArrowFunctionExpression6.errors.txt
diff --git a/tests/baselines/reference/parserArrowFunctionExpression6.js b/tests/baselines/reference/parserArrowFunctionExpression6.js
@@ -6,7 +6,6 @@ function foo(q: string, b: number) {
 
 //// [parserArrowFunctionExpression6.js]
 function foo(q, b) {
-    return true ? function (q, , ) { } : ;
-    ;
+    return true ? (q ? true : false) : (b = q.length, function () { });
 }
 ;
diff --git a/tests/baselines/reference/parserArrowFunctionExpression6.symbols b/tests/baselines/reference/parserArrowFunctionExpression6.symbols
@@ -5,11 +5,11 @@ function foo(q: string, b: number) {
 >b : Symbol(b, Decl(parserArrowFunctionExpression6.ts, 0, 23))
 
     return true ? (q ? true : false) : (b = q.length, function() { });
->q : Symbol(q, Decl(parserArrowFunctionExpression6.ts, 1, 19))
-> : Symbol((Missing), Decl(parserArrowFunctionExpression6.ts, 1, 22))
-> : Symbol((Missing), Decl(parserArrowFunctionExpression6.ts, 1, 29))
->b : Symbol(b, Decl(parserArrowFunctionExpression6.ts, 1, 40))
->q : Symbol(q, Decl(parserArrowFunctionExpression6.ts, 1, 19))
+>q : Symbol(q, Decl(parserArrowFunctionExpression6.ts, 0, 13))
+>b : Symbol(b, Decl(parserArrowFunctionExpression6.ts, 0, 23))
+>q.length : Symbol(String.length, Decl(lib.d.ts, --, --))
+>q : Symbol(q, Decl(parserArrowFunctionExpression6.ts, 0, 13))
+>length : Symbol(String.length, Decl(lib.d.ts, --, --))
 
 };
 
diff --git a/tests/baselines/reference/parserArrowFunctionExpression6.types b/tests/baselines/reference/parserArrowFunctionExpression6.types

Original file line number	Diff line number	Diff line change
`@@ -1835,7 +1835,7 @@ namespace ts {`
`1835`	`1835`	`}`
`1836`	`1836`
`1837`	`1837`	`function checkStrictModeNumericLiteral(node: NumericLiteral) {`
`1838`		`- if (inStrictMode && node.numericLiteralFlags & NumericLiteralFlags.Octal) {`
	`1838`	`+ if (inStrictMode && node.numericLiteralFlags & TokenFlags.Octal) {`
`1839`	`1839`	`file.bindDiagnostics.push(createDiagnosticForNode(node, Diagnostics.Octal_literals_are_not_allowed_in_strict_mode));`
`1840`	`1840`	`}`
`1841`	`1841`	`}`
`@@ -3319,7 +3319,7 @@ namespace ts {`
`3319`	`3319`	`break;`
`3320`	`3320`
`3321`	`3321`	`case SyntaxKind.NumericLiteral:`
`3322`		`- if ((<NumericLiteral>node).numericLiteralFlags & NumericLiteralFlags.BinaryOrOctalSpecifier) {`
	`3322`	`+ if ((<NumericLiteral>node).numericLiteralFlags & TokenFlags.BinaryOrOctalSpecifier) {`
`3323`	`3323`	`transformFlags \|= TransformFlags.AssertES2015;`
`3324`	`3324`	`}`
`3325`	`3325`	`break;`
Original file line number	Diff line number	Diff line change
`@@ -3633,7 +3633,7 @@ namespace ts {`
`3633`	`3633`	`* @param node A string literal.`
`3634`	`3634`	`*/`
`3635`	`3635`	`function visitNumericLiteral(node: NumericLiteral) {`
`3636`		`- if (node.numericLiteralFlags & NumericLiteralFlags.BinaryOrOctalSpecifier) {`
	`3636`	`+ if (node.numericLiteralFlags & TokenFlags.BinaryOrOctalSpecifier) {`
`3637`	`3637`	`return setTextRange(createNumericLiteral(node.text), node);`
`3638`	`3638`	`}`
`3639`	`3639`	`return node;`
Original file line number	Diff line number	Diff line change
`@@ -10,7 +10,7 @@ var X = { 0b11: '', 3: '' };`
`10`	`10`	`//// [duplicateIdentifierDifferentSpelling.js]`
`11`	`11`	`var A = /** @class */ (function () {`
`12`	`12`	`function A() {`
`13`		`- this[0b11] = '';`
	`13`	`+ this[3] = '';`
`14`	`14`	`this[3] = '';`
`15`	`15`	`}`
`16`	`16`	`return A;`
Original file line number	Diff line number	Diff line change
`@@ -6,7 +6,6 @@ function foo(q: string, b: number) {`
`6`	`6`
`7`	`7`	`//// [parserArrowFunctionExpression6.js]`
`8`	`8`	`function foo(q, b) {`
`9`		`- return true ? function (q, , ) { } : ;`
`10`		`- ;`
	`9`	`+ return true ? (q ? true : false) : (b = q.length, function () { });`
`11`	`10`	`}`
`12`	`11`	`;`