Introduce TokenFlags enum

ahejlsberg · ahejlsberg · commit ddf0df9cbb92 · 2017-11-12T09:11:09.000-08:00
diff --git a/src/compiler/binder.ts b/src/compiler/binder.ts
@@ -1835,7 +1835,7 @@ namespace ts {
         }
 
         function checkStrictModeNumericLiteral(node: NumericLiteral) {
-            if (inStrictMode && node.numericLiteralFlags & NumericLiteralFlags.Octal) {
+            if (inStrictMode && node.numericLiteralFlags & TokenFlags.Octal) {
                 file.bindDiagnostics.push(createDiagnosticForNode(node, Diagnostics.Octal_literals_are_not_allowed_in_strict_mode));
             }
         }
@@ -3319,7 +3319,7 @@ namespace ts {
                 break;
 
             case SyntaxKind.NumericLiteral:
-                if ((<NumericLiteral>node).numericLiteralFlags & NumericLiteralFlags.BinaryOrOctalSpecifier) {
+                if ((<NumericLiteral>node).numericLiteralFlags & TokenFlags.BinaryOrOctalSpecifier) {
                     transformFlags |= TransformFlags.AssertES2015;
                 }
                 break;
diff --git a/src/compiler/checker.ts b/src/compiler/checker.ts
@@ -25779,7 +25779,7 @@ namespace ts {
 
         function checkGrammarNumericLiteral(node: NumericLiteral): boolean {
             // Grammar checking
-            if (node.numericLiteralFlags & NumericLiteralFlags.Octal) {
+            if (node.numericLiteralFlags & TokenFlags.Octal) {
                 let diagnosticMessage: DiagnosticMessage | undefined;
                 if (languageVersion >= ScriptTarget.ES5) {
                     diagnosticMessage = Diagnostics.Octal_literals_are_not_available_when_targeting_ECMAScript_5_and_higher_Use_the_syntax_0;
diff --git a/src/compiler/parser.ts b/src/compiler/parser.ts
@@ -2111,7 +2111,7 @@ namespace ts {
             // We also do not need to check for negatives because any prefix operator would be part of a
             // parent unary expression.
             if (node.kind === SyntaxKind.NumericLiteral) {
-                (<NumericLiteral>node).numericLiteralFlags = scanner.getNumericLiteralFlags();
+                (<NumericLiteral>node).numericLiteralFlags = scanner.getTokenFlags() & TokenFlags.NumericLiteralFlags;
             }
 
             nextToken();
diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts
@@ -27,7 +27,7 @@ namespace ts {
         isReservedWord(): boolean;
         isUnterminated(): boolean;
         /* @internal */
-        getNumericLiteralFlags(): NumericLiteralFlags;
+        getTokenFlags(): TokenFlags;
         reScanGreaterToken(): SyntaxKind;
         reScanSlashToken(): SyntaxKind;
         reScanTemplateToken(): SyntaxKind;
@@ -814,10 +814,7 @@ namespace ts {
 
         let token: SyntaxKind;
         let tokenValue: string;
-        let precedingLineBreak: boolean;
-        let hasExtendedUnicodeEscape: boolean;
-        let tokenIsUnterminated: boolean;
-        let numericLiteralFlags: NumericLiteralFlags;
+        let tokenFlags: TokenFlags;
 
         setText(text, start, length);
 
@@ -828,12 +825,12 @@ namespace ts {
             getTokenPos: () => tokenPos,
             getTokenText: () => text.substring(tokenPos, pos),
             getTokenValue: () => tokenValue,
-            hasExtendedUnicodeEscape: () => hasExtendedUnicodeEscape,
-            hasPrecedingLineBreak: () => precedingLineBreak,
+            hasExtendedUnicodeEscape: () => (tokenFlags & TokenFlags.ExtendedUnicodeEscape) !== 0,
+            hasPrecedingLineBreak: () => (tokenFlags & TokenFlags.PrecedingLineBreak) !== 0,
             isIdentifier: () => token === SyntaxKind.Identifier || token > SyntaxKind.LastReservedWord,
             isReservedWord: () => token >= SyntaxKind.FirstReservedWord && token <= SyntaxKind.LastReservedWord,
-            isUnterminated: () => tokenIsUnterminated,
-            getNumericLiteralFlags: () => numericLiteralFlags,
+            isUnterminated: () => (tokenFlags & TokenFlags.Unterminated) !== 0,
+            getTokenFlags: () => tokenFlags,
             reScanGreaterToken,
             reScanSlashToken,
             reScanTemplateToken,
@@ -870,7 +867,7 @@ namespace ts {
             let end = pos;
             if (text.charCodeAt(pos) === CharacterCodes.E || text.charCodeAt(pos) === CharacterCodes.e) {
                 pos++;
-                numericLiteralFlags = NumericLiteralFlags.Scientific;
+                tokenFlags |= TokenFlags.Scientific;
                 if (text.charCodeAt(pos) === CharacterCodes.plus || text.charCodeAt(pos) === CharacterCodes.minus) pos++;
                 if (isDigit(text.charCodeAt(pos))) {
                     pos++;
@@ -942,7 +939,7 @@ namespace ts {
             while (true) {
                 if (pos >= end) {
                     result += text.substring(start, pos);
-                    tokenIsUnterminated = true;
+                    tokenFlags |= TokenFlags.Unterminated;
                     error(Diagnostics.Unterminated_string_literal);
                     break;
                 }
@@ -960,7 +957,7 @@ namespace ts {
                 }
                 if (isLineBreak(ch)) {
                     result += text.substring(start, pos);
-                    tokenIsUnterminated = true;
+                    tokenFlags |= TokenFlags.Unterminated;
                     error(Diagnostics.Unterminated_string_literal);
                     break;
                 }
@@ -984,7 +981,7 @@ namespace ts {
             while (true) {
                 if (pos >= end) {
                     contents += text.substring(start, pos);
-                    tokenIsUnterminated = true;
+                    tokenFlags |= TokenFlags.Unterminated;
                     error(Diagnostics.Unterminated_template_literal);
                     resultingToken = startedWithBacktick ? SyntaxKind.NoSubstitutionTemplateLiteral : SyntaxKind.TemplateTail;
                     break;
@@ -1070,7 +1067,7 @@ namespace ts {
                 case CharacterCodes.u:
                     // '\u{DDDDDDDD}'
                     if (pos < end && text.charCodeAt(pos) === CharacterCodes.openBrace) {
-                        hasExtendedUnicodeEscape = true;
+                        tokenFlags |= TokenFlags.ExtendedUnicodeEscape;
                         pos++;
                         return scanExtendedUnicodeEscape();
                     }
@@ -1239,10 +1236,7 @@ namespace ts {
 
         function scan(): SyntaxKind {
             startPos = pos;
-            hasExtendedUnicodeEscape = false;
-            precedingLineBreak = false;
-            tokenIsUnterminated = false;
-            numericLiteralFlags = 0;
+            tokenFlags = 0;
             while (true) {
                 tokenPos = pos;
                 if (pos >= end) {
@@ -1264,7 +1258,7 @@ namespace ts {
                 switch (ch) {
                     case CharacterCodes.lineFeed:
                     case CharacterCodes.carriageReturn:
-                        precedingLineBreak = true;
+                        tokenFlags |= TokenFlags.PrecedingLineBreak;
                         if (skipTrivia) {
                             pos++;
                             continue;
@@ -1407,7 +1401,7 @@ namespace ts {
                                 }
 
                                 if (isLineBreak(ch)) {
-                                    precedingLineBreak = true;
+                                    tokenFlags |= TokenFlags.PrecedingLineBreak;
                                 }
                                 pos++;
                             }
@@ -1420,7 +1414,9 @@ namespace ts {
                                 continue;
                             }
                             else {
-                                tokenIsUnterminated = !commentClosed;
+                                if (!commentClosed) {
+                                    tokenFlags |= TokenFlags.Unterminated;
+                                }
                                 return token = SyntaxKind.MultiLineCommentTrivia;
                             }
                         }
@@ -1441,7 +1437,7 @@ namespace ts {
                                 value = 0;
                             }
                             tokenValue = "" + value;
-                            numericLiteralFlags = NumericLiteralFlags.HexSpecifier;
+                            tokenFlags |= TokenFlags.HexSpecifier;
                             return token = SyntaxKind.NumericLiteral;
                         }
                         else if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.B || text.charCodeAt(pos + 1) === CharacterCodes.b)) {
@@ -1452,7 +1448,7 @@ namespace ts {
                                 value = 0;
                             }
                             tokenValue = "" + value;
-                            numericLiteralFlags = NumericLiteralFlags.BinarySpecifier;
+                            tokenFlags |= TokenFlags.BinarySpecifier;
                             return token = SyntaxKind.NumericLiteral;
                         }
                         else if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.O || text.charCodeAt(pos + 1) === CharacterCodes.o)) {
@@ -1463,13 +1459,13 @@ namespace ts {
                                 value = 0;
                             }
                             tokenValue = "" + value;
-                            numericLiteralFlags = NumericLiteralFlags.OctalSpecifier;
+                            tokenFlags |= TokenFlags.OctalSpecifier;
                             return token = SyntaxKind.NumericLiteral;
                         }
                         // Try to parse as an octal
                         if (pos + 1 < end && isOctalDigit(text.charCodeAt(pos + 1))) {
                             tokenValue = "" + scanOctalDigits();
-                            numericLiteralFlags = NumericLiteralFlags.Octal;
+                            tokenFlags |= TokenFlags.Octal;
                             return token = SyntaxKind.NumericLiteral;
                         }
                         // This fall-through is a deviation from the EcmaScript grammar. The grammar says that a leading zero
@@ -1626,7 +1622,7 @@ namespace ts {
                             continue;
                         }
                         else if (isLineBreak(ch)) {
-                            precedingLineBreak = true;
+                            tokenFlags |= TokenFlags.PrecedingLineBreak;
                             pos++;
                             continue;
                         }
@@ -1669,14 +1665,14 @@ namespace ts {
                     // If we reach the end of a file, or hit a newline, then this is an unterminated
                     // regex.  Report error and return what we have so far.
                     if (p >= end) {
-                        tokenIsUnterminated = true;
+                        tokenFlags |= TokenFlags.Unterminated;
                         error(Diagnostics.Unterminated_regular_expression_literal);
                         break;
                     }
 
                     const ch = text.charCodeAt(p);
                     if (isLineBreak(ch)) {
-                        tokenIsUnterminated = true;
+                        tokenFlags |= TokenFlags.Unterminated;
                         error(Diagnostics.Unterminated_regular_expression_literal);
                         break;
                     }
@@ -1894,7 +1890,7 @@ namespace ts {
             const saveTokenPos = tokenPos;
             const saveToken = token;
             const saveTokenValue = tokenValue;
-            const savePrecedingLineBreak = precedingLineBreak;
+            const saveTokenFlags = tokenFlags;
             const result = callback();
 
             // If our callback returned something 'falsy' or we're just looking ahead,
@@ -1905,7 +1901,7 @@ namespace ts {
                 tokenPos = saveTokenPos;
                 token = saveToken;
                 tokenValue = saveTokenValue;
-                precedingLineBreak = savePrecedingLineBreak;
+                tokenFlags = saveTokenFlags;
             }
             return result;
         }
@@ -1916,10 +1912,8 @@ namespace ts {
             const saveStartPos = startPos;
             const saveTokenPos = tokenPos;
             const saveToken = token;
-            const savePrecedingLineBreak = precedingLineBreak;
             const saveTokenValue = tokenValue;
-            const saveHasExtendedUnicodeEscape = hasExtendedUnicodeEscape;
-            const saveTokenIsUnterminated = tokenIsUnterminated;
+            const saveTokenFlags = tokenFlags;
 
             setText(text, start, length);
             const result = callback();
@@ -1929,10 +1923,8 @@ namespace ts {
             startPos = saveStartPos;
             tokenPos = saveTokenPos;
             token = saveToken;
-            precedingLineBreak = savePrecedingLineBreak;
             tokenValue = saveTokenValue;
-            hasExtendedUnicodeEscape = saveHasExtendedUnicodeEscape;
-            tokenIsUnterminated = saveTokenIsUnterminated;
+            tokenFlags = saveTokenFlags;
 
             return result;
         }
@@ -1973,11 +1965,8 @@ namespace ts {
             startPos = textPos;
             tokenPos = textPos;
             token = SyntaxKind.Unknown;
-            precedingLineBreak = false;
-
             tokenValue = undefined;
-            hasExtendedUnicodeEscape = false;
-            tokenIsUnterminated = false;
+            tokenFlags = 0;
         }
     }
 }
diff --git a/src/compiler/transformers/es2015.ts b/src/compiler/transformers/es2015.ts
@@ -3633,7 +3633,7 @@ namespace ts {
          * @param node A string literal.
          */
         function visitNumericLiteral(node: NumericLiteral) {
-            if (node.numericLiteralFlags & NumericLiteralFlags.BinaryOrOctalSpecifier) {
+            if (node.numericLiteralFlags & TokenFlags.BinaryOrOctalSpecifier) {
                 return setTextRange(createNumericLiteral(node.text), node);
             }
             return node;
diff --git a/src/compiler/types.ts b/src/compiler/types.ts
@@ -1451,20 +1451,25 @@ namespace ts {
     }
 
     /* @internal */
-    export const enum NumericLiteralFlags {
+    export const enum TokenFlags {
         None = 0,
-        Scientific = 1 << 1,        // e.g. `10e2`
-        Octal = 1 << 2,             // e.g. `0777`
-        HexSpecifier = 1 << 3,      // e.g. `0x00000000`
-        BinarySpecifier = 1 << 4,   // e.g. `0b0110010000000000`
-        OctalSpecifier = 1 << 5,    // e.g. `0o777`
+        PrecedingLineBreak = 1 << 0,
+        PrecedingComment = 1 << 1,
+        Unterminated = 1 << 2,
+        ExtendedUnicodeEscape = 1 << 3,
+        Scientific = 1 << 4,        // e.g. `10e2`
+        Octal = 1 << 5,             // e.g. `0777`
+        HexSpecifier = 1 << 6,      // e.g. `0x00000000`
+        BinarySpecifier = 1 << 7,   // e.g. `0b0110010000000000`
+        OctalSpecifier = 1 << 8,    // e.g. `0o777`
         BinaryOrOctalSpecifier = BinarySpecifier | OctalSpecifier,
+        NumericLiteralFlags = Scientific | Octal | HexSpecifier | BinarySpecifier | OctalSpecifier
     }
 
     export interface NumericLiteral extends LiteralExpression {
         kind: SyntaxKind.NumericLiteral;
         /* @internal */
-        numericLiteralFlags?: NumericLiteralFlags;
+        numericLiteralFlags?: TokenFlags;
     }
 
     export interface TemplateHead extends LiteralLikeNode {

Original file line number	Diff line number	Diff line change
`@@ -1835,7 +1835,7 @@ namespace ts {`
`1835`	`1835`	`}`
`1836`	`1836`
`1837`	`1837`	`function checkStrictModeNumericLiteral(node: NumericLiteral) {`
`1838`		`- if (inStrictMode && node.numericLiteralFlags & NumericLiteralFlags.Octal) {`
	`1838`	`+ if (inStrictMode && node.numericLiteralFlags & TokenFlags.Octal) {`
`1839`	`1839`	`file.bindDiagnostics.push(createDiagnosticForNode(node, Diagnostics.Octal_literals_are_not_allowed_in_strict_mode));`
`1840`	`1840`	`}`
`1841`	`1841`	`}`
`@@ -3319,7 +3319,7 @@ namespace ts {`
`3319`	`3319`	`break;`
`3320`	`3320`
`3321`	`3321`	`case SyntaxKind.NumericLiteral:`
`3322`		`- if ((<NumericLiteral>node).numericLiteralFlags & NumericLiteralFlags.BinaryOrOctalSpecifier) {`
	`3322`	`+ if ((<NumericLiteral>node).numericLiteralFlags & TokenFlags.BinaryOrOctalSpecifier) {`
`3323`	`3323`	`transformFlags \|= TransformFlags.AssertES2015;`
`3324`	`3324`	`}`
`3325`	`3325`	`break;`
Original file line number	Diff line number	Diff line change
`@@ -2111,7 +2111,7 @@ namespace ts {`
`2111`	`2111`	`// We also do not need to check for negatives because any prefix operator would be part of a`
`2112`	`2112`	`// parent unary expression.`
`2113`	`2113`	`if (node.kind === SyntaxKind.NumericLiteral) {`
`2114`		`- (<NumericLiteral>node).numericLiteralFlags = scanner.getNumericLiteralFlags();`
	`2114`	`+ (<NumericLiteral>node).numericLiteralFlags = scanner.getTokenFlags() & TokenFlags.NumericLiteralFlags;`
`2115`	`2115`	`}`
`2116`	`2116`
`2117`	`2117`	`nextToken();`
Original file line number	Diff line number	Diff line change
`@@ -3633,7 +3633,7 @@ namespace ts {`
`3633`	`3633`	`* @param node A string literal.`
`3634`	`3634`	`*/`
`3635`	`3635`	`function visitNumericLiteral(node: NumericLiteral) {`
`3636`		`- if (node.numericLiteralFlags & NumericLiteralFlags.BinaryOrOctalSpecifier) {`
	`3636`	`+ if (node.numericLiteralFlags & TokenFlags.BinaryOrOctalSpecifier) {`
`3637`	`3637`	`return setTextRange(createNumericLiteral(node.text), node);`
`3638`	`3638`	`}`
`3639`	`3639`	`return node;`