Support extended unicode escapes in identifiers, per es6 spec (microsoft#32725)

weswigham · web-flow · commit d00056f096bf · 2019-08-06T14:43:41.000-07:00
diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts
@@ -1342,6 +1342,19 @@ namespace ts {
             return -1;
         }
 
+
+        function peekExtendedUnicodeEscape(): number {
+            if (languageVersion >= ScriptTarget.ES2015 && codePointAt(text, pos + 1) === CharacterCodes.u && codePointAt(text, pos + 2) === CharacterCodes.openBrace) {
+                const start = pos;
+                pos += 3;
+                const escapedValueString = scanMinimumNumberOfHexDigits(1, /*canHaveSeparators*/ false);
+                const escapedValue = escapedValueString ? parseInt(escapedValueString, 16) : -1;
+                pos = start;
+                return escapedValue;
+            }
+            return -1;
+        }
+
         function scanIdentifierParts(): string {
             let result = "";
             let start = pos;
@@ -1351,6 +1364,14 @@ namespace ts {
                     pos += charSize(ch);
                 }
                 else if (ch === CharacterCodes.backslash) {
+                    ch = peekExtendedUnicodeEscape();
+                    if (ch >= 0 && isIdentifierPart(ch, languageVersion)) {
+                        pos += 3;
+                        tokenFlags |= TokenFlags.ExtendedUnicodeEscape;
+                        result += scanExtendedUnicodeEscape();
+                        start = pos;
+                        continue;
+                    }
                     ch = peekUnicodeEscape();
                     if (!(ch >= 0 && isIdentifierPart(ch, languageVersion))) {
                         break;
@@ -1836,12 +1857,21 @@ namespace ts {
                         pos++;
                         return token = SyntaxKind.AtToken;
                     case CharacterCodes.backslash:
+                        const extendedCookedChar = peekExtendedUnicodeEscape();
+                        if (extendedCookedChar >= 0 && isIdentifierStart(extendedCookedChar, languageVersion)) {
+                            pos += 3;
+                            tokenFlags |= TokenFlags.ExtendedUnicodeEscape;
+                            tokenValue = scanExtendedUnicodeEscape() + scanIdentifierParts();
+                            return token = getIdentifierToken();
+                        }
+
                         const cookedChar = peekUnicodeEscape();
                         if (cookedChar >= 0 && isIdentifierStart(cookedChar, languageVersion)) {
                             pos += 6;
                             tokenValue = String.fromCharCode(cookedChar) + scanIdentifierParts();
                             return token = getIdentifierToken();
                         }
+
                         error(Diagnostics.Invalid_character);
                         pos++;
                         return token = SyntaxKind.Unknown;
diff --git a/tests/baselines/reference/extendedUnicodeEscapeSequenceIdentifiers.js b/tests/baselines/reference/extendedUnicodeEscapeSequenceIdentifiers.js
@@ -0,0 +1,11 @@
+//// [extendedUnicodeEscapeSequenceIdentifiers.ts]
+const \u{0061} = 12;
+const a\u{0061} = 12;
+
+console.log(a + aa);
+
+
+//// [extendedUnicodeEscapeSequenceIdentifiers.js]
+const \u{0061} = 12;
+const a\u{0061} = 12;
+console.log(a + aa);
diff --git a/tests/baselines/reference/extendedUnicodeEscapeSequenceIdentifiers.symbols b/tests/baselines/reference/extendedUnicodeEscapeSequenceIdentifiers.symbols
@@ -0,0 +1,14 @@
+=== tests/cases/compiler/extendedUnicodeEscapeSequenceIdentifiers.ts ===
+const \u{0061} = 12;
+>\u{0061} : Symbol(\u{0061}, Decl(extendedUnicodeEscapeSequenceIdentifiers.ts, 0, 5))
+
+const a\u{0061} = 12;
+>a\u{0061} : Symbol(a\u{0061}, Decl(extendedUnicodeEscapeSequenceIdentifiers.ts, 1, 5))
+
+console.log(a + aa);
+>console.log : Symbol(Console.log, Decl(lib.dom.d.ts, --, --))
+>console : Symbol(console, Decl(lib.dom.d.ts, --, --))
+>log : Symbol(Console.log, Decl(lib.dom.d.ts, --, --))
+>a : Symbol(\u{0061}, Decl(extendedUnicodeEscapeSequenceIdentifiers.ts, 0, 5))
+>aa : Symbol(a\u{0061}, Decl(extendedUnicodeEscapeSequenceIdentifiers.ts, 1, 5))
+
diff --git a/tests/baselines/reference/extendedUnicodeEscapeSequenceIdentifiers.types b/tests/baselines/reference/extendedUnicodeEscapeSequenceIdentifiers.types
@@ -0,0 +1,18 @@
+=== tests/cases/compiler/extendedUnicodeEscapeSequenceIdentifiers.ts ===
+const \u{0061} = 12;
+>\u{0061} : 12
+>12 : 12
+
+const a\u{0061} = 12;
+>a\u{0061} : 12
+>12 : 12
+
+console.log(a + aa);
+>console.log(a + aa) : void
+>console.log : (message?: any, ...optionalParams: any[]) => void
+>console : Console
+>log : (message?: any, ...optionalParams: any[]) => void
+>a + aa : number
+>a : 12
+>aa : 12
+
diff --git a/tests/cases/compiler/extendedUnicodeEscapeSequenceIdentifiers.ts b/tests/cases/compiler/extendedUnicodeEscapeSequenceIdentifiers.ts
@@ -0,0 +1,5 @@
+// @target: es6
+const \u{0061} = 12;
+const a\u{0061} = 12;
+
+console.log(a + aa);
