From 154f10e3fda8360bcc969e794a106012f1abbbe3 Mon Sep 17 00:00:00 2001 From: Keith Hill Date: Sat, 3 Jun 2017 00:25:18 -0600 Subject: [PATCH 1/7] Implement Unicode fixed width `uXXXX and variable width `u{xxxxxx} escape sequences. --- .../engine/parser/tokenizer.cs | 175 ++++++++++++++++-- .../resources/ParserStrings.resx | 79 ++++---- .../Language/Parser/Parser.Tests.ps1 | 78 +++++++- .../Language/Parser/Parsing.Tests.ps1 | 15 ++ 4 files changed, 298 insertions(+), 49 deletions(-) diff --git a/src/System.Management.Automation/engine/parser/tokenizer.cs b/src/System.Management.Automation/engine/parser/tokenizer.cs index 765353a2593..85935f2429c 100644 --- a/src/System.Management.Automation/engine/parser/tokenizer.cs +++ b/src/System.Management.Automation/engine/parser/tokenizer.cs @@ -507,6 +507,10 @@ private static readonly Dictionary s_keywordTable private static readonly Dictionary s_operatorTable = new Dictionary(StringComparer.OrdinalIgnoreCase); + // Unicode replacement char used to represent an unknown, unrecognized or unrepresentable character in a + // Unicode escape sequence. + private static readonly string s_unknownUnicodeChar = ((char)0xffdd).ToString(); + private readonly Parser _parser; private PositionHelper _positionHelper; private int _nestedTokensAdjustment; @@ -1229,23 +1233,134 @@ private int SkipBlockComment(int i) return i; } - private static char Backtick(char c) + private string Backtick(char c) { switch (c) { - case '0': return '\0'; - case 'a': return '\a'; - case 'b': return '\b'; - case 'e': return '\u001b'; - case 'f': return '\f'; - case 'n': return '\n'; - case 'r': return '\r'; - case 't': return '\t'; - case 'v': return '\v'; - default: return c; + case '0': return "\0"; + case 'a': return "\a"; + case 'b': return "\b"; + case 'e': return "\u001b"; + case 'f': return "\f"; + case 'n': return "\n"; + case 'r': return "\r"; + case 't': return "\t"; + case 'u': return ScanUnicodeEscapeSequence(); + case 'v': return "\v"; + default: return c.ToString(); } } + private string ScanUnicodeEscapeSequence() + { + int escSeqStartIndex = _currentIndex - 2; + bool isBracketedSequence = false; + bool isTerminated = false; + + char c = PeekChar(); + if (c == '{') + { + SkipChar(); + isBracketedSequence = true; + } + + int maxNumberOfHexDigits = (isBracketedSequence ? 6 : 4); + + // Scan hex chars after the Unicode escape sequence start. + var sb = GetStringBuilder(); + int i; + for (i = 0; i < maxNumberOfHexDigits; i++) + { + c = GetChar(); + + // Bracketed sequence has been terminated. + if (isBracketedSequence && (c == '}')) + { + if (i == 0) + { + // Variable sequence must have at least one hex char. + IScriptExtent errorExtent = NewScriptExtent(escSeqStartIndex, _currentIndex); + ReportError(errorExtent, () => ParserStrings.EmptyUnicodeEscapeSequence); + return s_unknownUnicodeChar; + } + + isTerminated = true; + break; + } + + if (!c.IsHexDigit()) + { + UngetChar(); + + if (isBracketedSequence) + { + ReportError(_currentIndex, () => ParserStrings.MissingUnicodeEscapeSequenceTerminator); + } + else + { + IScriptExtent errorExtent = NewScriptExtent(escSeqStartIndex, _currentIndex); + ReportError(errorExtent, () => ParserStrings.InvalidUnicodeEscapeSequence); + } + + return s_unknownUnicodeChar; + } + + sb.Append(c); + } + + if (isBracketedSequence && !isTerminated) + { + c = GetChar(); + if (c != '}') + { + UngetChar(); + ReportError(_currentIndex, + i == maxNumberOfHexDigits && c.IsHexDigit() + ? (Expression>)(() => ParserStrings.TooManyDigitsInUnicodeEscapeSequence) + : () => ParserStrings.MissingUnicodeEscapeSequenceTerminator); + return s_unknownUnicodeChar; + } + } + + string hexStr = GetStringAndRelease(sb); + + int unicodeValue = int.Parse(hexStr, NumberStyles.AllowHexSpecifier, NumberFormatInfo.InvariantInfo); + if (unicodeValue <= Char.MaxValue) + { + return ((char)unicodeValue).ToString(); + } + else if (unicodeValue <= 0x10FFFF) + { + return Char.ConvertFromUtf32(unicodeValue); + } + else + { + // Place the error indicator under only the hex digits in the esc sequence. + IScriptExtent errorExtent = NewScriptExtent(escSeqStartIndex + 3, _currentIndex - 1); + ReportError(errorExtent, () => ParserStrings.InvalidUnicodeEscapeSequenceValue); + return s_unknownUnicodeChar; + } + } + + private bool IsSurrogatePair(string str, ref char nonSurrogatePairChar, StringBuilder sb1 = null, StringBuilder sb2 = null) + { + Diagnostics.Assert(!string.IsNullOrEmpty(str), "Caller never calls us with an empty string"); + Diagnostics.Assert((str.Length <= 2), "Caller never calls us with a string length greater than two"); + Diagnostics.Assert((str.Length == 1) || Char.IsSurrogate(str, 0), + "Caller never calls us with a string length of two and not be a surrogate pair"); + + if (str.Length == 1) + { + nonSurrogatePairChar = str[0]; + return false; + } + + sb1?.Append(str); + sb2?.Append(str); + + return true; + } + private void ScanToEndOfCommentLine(out bool sawBeginSig, out bool matchedRequires) { // When we get here, we are scanning a line comment. To avoid rescanning, @@ -2029,7 +2144,12 @@ private TokenFlags ScanStringExpandable(StringBuilder sb, StringBuilder formatSb if (c1 != 0) { SkipChar(); - c = Backtick(c1); + string str = Backtick(c1); + if (IsSurrogatePair(str, ref c, sb, formatSb)) + { + // The character has been processed and appended to the string builders + continue; + } } } if (c == '{' || c == '}') @@ -2338,7 +2458,12 @@ private Token ScanHereStringExpandable() if (c1 != 0) { SkipChar(); - c = Backtick(c1); + string str = Backtick(c1); + if (IsSurrogatePair(str, ref c, sb, formatSb)) + { + // The character has been processed and appended to the string builders + continue; + } } } if (c == '{' || c == '}') @@ -2407,7 +2532,12 @@ private Token ScanVariable(bool splatted, bool inStringExpandable) UngetChar(); goto end_braced_variable_scan; } - c = Backtick(c1); + string str = Backtick(c1); + if (IsSurrogatePair(str, ref c, sb)) + { + // The character has been processed and appended to the string builder + continue; + } break; } case '"': @@ -2845,6 +2975,13 @@ private Token ScanGenericToken(char firstChar) return ScanGenericToken(sb); } + private Token ScanGenericToken(string str) + { + var sb = GetStringBuilder(); + sb.Append(str); + return ScanGenericToken(sb); + } + private Token ScanGenericToken(StringBuilder sb) { // On entry, we've already scanned an unknown number of characters @@ -2871,6 +3008,7 @@ private Token ScanGenericToken(StringBuilder sb) // Make sure our token does not start with any of these characters. //Contract.Requires(Contract.ForAll("{}()@#;,|&\r\n\t ", c1 => sb[0] != c1)); + string str; List nestedTokens = new List(); var formatSb = GetStringBuilder(); formatSb.Append(sb); @@ -2885,7 +3023,12 @@ private Token ScanGenericToken(StringBuilder sb) if (c1 != 0) { SkipChar(); - c = Backtick(c1); + str = Backtick(c1); + if (IsSurrogatePair(str, ref c, sb, formatSb)) + { + // The character has been processed and appended to the string builders + continue; + } } } else if (c.IsSingleQuote()) @@ -2919,7 +3062,7 @@ private Token ScanGenericToken(StringBuilder sb) } UngetChar(); - var str = GetStringAndRelease(sb); + str = GetStringAndRelease(sb); if (nestedTokens.Count > 0) { return NewGenericExpandableToken(str, GetStringAndRelease(formatSb), nestedTokens); diff --git a/src/System.Management.Automation/resources/ParserStrings.resx b/src/System.Management.Automation/resources/ParserStrings.resx index 71dc1003989..73d747d4f3b 100644 --- a/src/System.Management.Automation/resources/ParserStrings.resx +++ b/src/System.Management.Automation/resources/ParserStrings.resx @@ -1,17 +1,17 @@  - @@ -129,6 +129,21 @@ Incomplete string token. + + An empty `u${} Unicode escape sequence was found. At least one hex digit is required inside the braces. + + + The Unicode escape sequence is not valid. A valid sequence is `u followed by four hex digits. + + + The Unicode escape sequence value is out of range. The maximum value is 0x10FFFF. + + + The Unicode escape sequence is missing the closing '}'. + + + The Unicode escape sequence contains more than the maximum of six hex digits between braces. + A number cannot be both a long and floating point. @@ -1039,10 +1054,10 @@ Possible matches are The script block that defines function '{0}' cannot be null or empty. Provide a non-empty script block in the function definition dictionary, and then try the operation again. - The syntax of the Import-DscResource dynamic keyword is: - + The syntax of the Import-DscResource dynamic keyword is: + Import-DscResource [-Name <ResourceName(s)>] [-ModuleName <ModuleName(s)>] [-ModuleVersion <ModuleVersion>]. - + Name : Names of one or more resources to import. ModuleName : Module names or ModuleSpecification objects of one or more modules to import. ModuleVersion : Version of module to import. If used, ModuleName must represent only one module by name. @@ -1072,7 +1087,7 @@ ModuleVersion : Version of module to import. If used, ModuleName must represent '{0}' is not a valid value for property '{1}' on class '{2}'. Please change the value to one of the following strings: {3}. - At least one of the values '{0}' is not supported or valid for property '{1}' on class '{2}'. Please specify only supported values: + At least one of the values '{0}' is not supported or valid for property '{1}' on class '{2}'. Please specify only supported values: {3}. @@ -1155,7 +1170,7 @@ ModuleVersion : Version of module to import. If used, ModuleName must represent Only one type may be specified on class members. - Error during creation of type "{0}". Error message: + Error during creation of type "{0}". Error message: {1} diff --git a/test/powershell/Language/Parser/Parser.Tests.ps1 b/test/powershell/Language/Parser/Parser.Tests.ps1 index 1a9f3938ca1..b81175961a3 100644 --- a/test/powershell/Language/Parser/Parser.Tests.ps1 +++ b/test/powershell/Language/Parser/Parser.Tests.ps1 @@ -1,4 +1,4 @@ -Describe "ParserTests (admin\monad\tests\monad\src\engine\core\ParserTests.cs)" -Tags "CI" { +Describe "ParserTests (admin\monad\tests\monad\src\engine\core\ParserTests.cs)" -Tags "CI" { BeforeAll { $functionDefinitionFile = Join-Path -Path $TestDrive -ChildPath "functionDefinition.ps1" $functionDefinition = @' @@ -270,6 +270,82 @@ Describe "ParserTests (admin\monad\tests\monad\src\engine\core\ParserTests.cs)" $result | should be ([char]0x1b) } + Context "Test Unicode escape sequences." { + # These tests require the file to be saved with a BOM. Unfortunately when this UTF8 file is read by + # PowerShell without a BOM, the file is incorrectly interpreted as ASCII. + It 'Test that Unicode escape sequence `u2195 in string returns the ↕ character.' { + $result = ExecuteCommand '"foo`u2195abc"' + $result | should be "foo↕abc" + } + + It 'Test that Unicode escape sequence `u2195 in here string returns the ↕ character.' { + $result = ExecuteCommand ("@`"`n`n" + 'foo`u2195abc' + "`n`n`"@") + $result | should be "`nfoo↕abc`n" + } + + It 'Test that the bracketed Unicode escape sequence `u{0} returns minimum char.' { + $result = ExecuteCommand '"`u{0}"' + [int]$result[0] | should be 0 + } + + It 'Test that the bracketed Unicode escape sequence `u{10FFFF} returns maximum surrogate char pair.' { + $result = ExecuteCommand '"`u{10FFFF}"' + [int]$result[0] | should be 0xDBFF # max value for high surrogate of surrogate pair + [int]$result[1] | should be 0xDFFF # max value for low surrogate of surrogate pair + } + + It 'Test that the bracketed Unicode escape sequence `u{a9} returns the © character.' { + $result = ExecuteCommand '"`u{a9}"' + $result | should be '©' + } + + It 'Test that the bracketed Unicode escape sequence `u{1f44d} returns surrogate pair for emoji 👍 character.' { + $result = ExecuteCommand '"`u{1f44d}"' + $result | should be "👍" + } + + It 'Test that Unicode escape sequence in single quoted is not processed.' { + $result = ExecuteCommand '''foo`u2195abc''' + $result | should be 'foo`u2195abc' + } + + It 'Test that Unicode escape sequence in single quoted here string is not processed.' { + $result = ExecuteCommand @" +@' + +foo``u2195abc + +'@ +"@ + $result | should be "`r`nfoo``u2195abc`r`n" + } + + It "Test that two consecutive Unicode escape sequences are tokenized correctly." { + $result = ExecuteCommand '"`u007b`u007d"' + $result | should be '{}' + } + + It "Test that a Unicode escape sequence can be used in a command name." { + function xyzzy`u2195($p) {$p} + $cmd = Get-Command xyzzy`u2195 -ErrorAction SilentlyContinue + $cmd | should not BeNullOrEmpty + $cmd.Name | should be 'xyzzy↕' + xyzzy`u2195 42 | should be 42 + } + + It "Test that a Unicode escape sequence can be used in a variable name." { + ${fooxyzzy`u{2195}} = 42 + $var = Get-Variable -Name fooxyzzy* -ErrorAction SilentlyContinue + $var | should not BeNullOrEmpty + $var.Name | should be "fooxyzzy↕" + $var.Value | should be 42 + } + + It "Test that a Unicode escape sequence can be used in an argument." { + Write-Output `u{a9}` Acme` Inc | should be "© Acme Inc" + } + } + It "Test that escaping any character with no special meaning just returns that char. (line 602)" { $result = ExecuteCommand '"fo`obar"' $result | should be "foobar" diff --git a/test/powershell/Language/Parser/Parsing.Tests.ps1 b/test/powershell/Language/Parser/Parsing.Tests.ps1 index 03034e3340a..3fcd0f8b22d 100644 --- a/test/powershell/Language/Parser/Parsing.Tests.ps1 +++ b/test/powershell/Language/Parser/Parsing.Tests.ps1 @@ -302,3 +302,18 @@ Describe 'expressions parsing' -Tags "CI" { Describe 'Hash Expression parsing' -Tags "CI" { ShouldBeParseError '@{ a=1;b=2;c=3;' MissingEndCurlyBrace 2 } + +Describe 'Unicode escape sequence parsing' -Tag "CI" { + ShouldBeParseError '"`u{110000}"' InvalidUnicodeEscapeSequenceValue 4 # error offset is "`u{>>1<<10000}" + ShouldBeParseError '"`u{1234567}"' TooManyDigitsInUnicodeEscapeSequence 10 # error offset is "`u{123456>>7<<}" + ShouldBeParseError '"`u219z"' InvalidUnicodeEscapeSequence 1 + ShouldBeParseError '"`u{219z"' MissingUnicodeEscapeSequenceTerminator 7 # error offset "`u{219>>z<<" + ShouldBeParseError '"`u{}"' EmptyUnicodeEscapeSequence 1 + ShouldBeParseError '"`u' InvalidUnicodeEscapeSequence,TerminatorExpectedAtEndOfString 1,0 + ShouldBeParseError '"`u123' InvalidUnicodeEscapeSequence,TerminatorExpectedAtEndOfString 1,0 + ShouldBeParseError '"`u219z' InvalidUnicodeEscapeSequence,TerminatorExpectedAtEndOfString 1,0 + ShouldBeParseError '"`u{' MissingUnicodeEscapeSequenceTerminator,TerminatorExpectedAtEndOfString 4,0 + ShouldBeParseError '"`u{1' MissingUnicodeEscapeSequenceTerminator,TerminatorExpectedAtEndOfString 5,0 + ShouldBeParseError '"`u{123456' MissingUnicodeEscapeSequenceTerminator,TerminatorExpectedAtEndOfString 10,0 + ShouldBeParseError '"`u{1234567' TooManyDigitsInUnicodeEscapeSequence,TerminatorExpectedAtEndOfString 10,0 +} From 9a0b8ab6420dbf8b8b902b47ce70fcf15068f14e Mon Sep 17 00:00:00 2001 From: Keith Hill Date: Thu, 22 Jun 2017 23:12:23 -0600 Subject: [PATCH 2/7] Remove fixed width Unicode escape sequence tokenizing. The variable width form will require an update to the EditorSyntax so editors don't show "unexpected token" errors when this form is used in a command or variable name. I will do that after this PR has been accepted unless the team prefer I submit a PR to that project before this PR is accepted. The same goes for the doc update to the about_Escape_Characters help topic. --- .../engine/parser/tokenizer.cs | 56 +++++++-------- .../resources/ParserStrings.resx | 69 +++++++++---------- .../Language/Parser/Parser.Tests.ps1 | 36 +++++----- .../Language/Parser/Parsing.Tests.ps1 | 9 +-- 4 files changed, 78 insertions(+), 92 deletions(-) diff --git a/src/System.Management.Automation/engine/parser/tokenizer.cs b/src/System.Management.Automation/engine/parser/tokenizer.cs index 85935f2429c..ad7482a850f 100644 --- a/src/System.Management.Automation/engine/parser/tokenizer.cs +++ b/src/System.Management.Automation/engine/parser/tokenizer.cs @@ -507,7 +507,7 @@ private static readonly Dictionary s_keywordTable private static readonly Dictionary s_operatorTable = new Dictionary(StringComparer.OrdinalIgnoreCase); - // Unicode replacement char used to represent an unknown, unrecognized or unrepresentable character in a + // Unicode replacement char used to represent an unknown, unrecognized or unrepresentable character in a // Unicode escape sequence. private static readonly string s_unknownUnicodeChar = ((char)0xffdd).ToString(); @@ -1254,17 +1254,18 @@ private string Backtick(char c) private string ScanUnicodeEscapeSequence() { int escSeqStartIndex = _currentIndex - 2; - bool isBracketedSequence = false; + int maxNumberOfHexDigits = 6; bool isTerminated = false; - char c = PeekChar(); - if (c == '{') + char c = GetChar(); + if (c != '{') { - SkipChar(); - isBracketedSequence = true; - } + UngetChar(); - int maxNumberOfHexDigits = (isBracketedSequence ? 6 : 4); + IScriptExtent errorExtent = NewScriptExtent(escSeqStartIndex, _currentIndex); + ReportError(errorExtent, () => ParserStrings.InvalidUnicodeEscapeSequence); + return s_unknownUnicodeChar; + } // Scan hex chars after the Unicode escape sequence start. var sb = GetStringBuilder(); @@ -1273,14 +1274,14 @@ private string ScanUnicodeEscapeSequence() { c = GetChar(); - // Bracketed sequence has been terminated. - if (isBracketedSequence && (c == '}')) + // Sequence has been terminated. + if (c == '}') { if (i == 0) { - // Variable sequence must have at least one hex char. + // Sequence must have at least one hex char. IScriptExtent errorExtent = NewScriptExtent(escSeqStartIndex, _currentIndex); - ReportError(errorExtent, () => ParserStrings.EmptyUnicodeEscapeSequence); + ReportError(errorExtent, () => ParserStrings.InvalidUnicodeEscapeSequence); return s_unknownUnicodeChar; } @@ -1292,23 +1293,14 @@ private string ScanUnicodeEscapeSequence() { UngetChar(); - if (isBracketedSequence) - { - ReportError(_currentIndex, () => ParserStrings.MissingUnicodeEscapeSequenceTerminator); - } - else - { - IScriptExtent errorExtent = NewScriptExtent(escSeqStartIndex, _currentIndex); - ReportError(errorExtent, () => ParserStrings.InvalidUnicodeEscapeSequence); - } - + ReportError(_currentIndex, () => ParserStrings.MissingUnicodeEscapeSequenceTerminator); return s_unknownUnicodeChar; } sb.Append(c); } - if (isBracketedSequence && !isTerminated) + if (!isTerminated) { c = GetChar(); if (c != '}') @@ -1342,14 +1334,14 @@ private string ScanUnicodeEscapeSequence() } } - private bool IsSurrogatePair(string str, ref char nonSurrogatePairChar, StringBuilder sb1 = null, StringBuilder sb2 = null) + private bool IsSurrogatePair(string str, ref char nonSurrogatePairChar, StringBuilder sb1 = null, StringBuilder sb2 = null) { Diagnostics.Assert(!string.IsNullOrEmpty(str), "Caller never calls us with an empty string"); Diagnostics.Assert((str.Length <= 2), "Caller never calls us with a string length greater than two"); - Diagnostics.Assert((str.Length == 1) || Char.IsSurrogate(str, 0), - "Caller never calls us with a string length of two and not be a surrogate pair"); - - if (str.Length == 1) + Diagnostics.Assert((str.Length == 1) || Char.IsSurrogate(str, 0), + "Caller never calls us with a string length of two and not be a surrogate pair"); + + if (str.Length == 1) { nonSurrogatePairChar = str[0]; return false; @@ -2149,7 +2141,7 @@ private TokenFlags ScanStringExpandable(StringBuilder sb, StringBuilder formatSb { // The character has been processed and appended to the string builders continue; - } + } } } if (c == '{' || c == '}') @@ -2463,7 +2455,7 @@ private Token ScanHereStringExpandable() { // The character has been processed and appended to the string builders continue; - } + } } } if (c == '{' || c == '}') @@ -2537,7 +2529,7 @@ private Token ScanVariable(bool splatted, bool inStringExpandable) { // The character has been processed and appended to the string builder continue; - } + } break; } case '"': @@ -3028,7 +3020,7 @@ private Token ScanGenericToken(StringBuilder sb) { // The character has been processed and appended to the string builders continue; - } + } } } else if (c.IsSingleQuote()) diff --git a/src/System.Management.Automation/resources/ParserStrings.resx b/src/System.Management.Automation/resources/ParserStrings.resx index 73d747d4f3b..3f43594b049 100644 --- a/src/System.Management.Automation/resources/ParserStrings.resx +++ b/src/System.Management.Automation/resources/ParserStrings.resx @@ -1,17 +1,17 @@  - @@ -129,11 +129,8 @@ Incomplete string token. - - An empty `u${} Unicode escape sequence was found. At least one hex digit is required inside the braces. - - The Unicode escape sequence is not valid. A valid sequence is `u followed by four hex digits. + The Unicode escape sequence is not valid. A valid sequence is `u{ followed by one to six hex digits and a closing '}'. The Unicode escape sequence value is out of range. The maximum value is 0x10FFFF. @@ -1054,10 +1051,10 @@ Possible matches are The script block that defines function '{0}' cannot be null or empty. Provide a non-empty script block in the function definition dictionary, and then try the operation again. - The syntax of the Import-DscResource dynamic keyword is: - + The syntax of the Import-DscResource dynamic keyword is: + Import-DscResource [-Name <ResourceName(s)>] [-ModuleName <ModuleName(s)>] [-ModuleVersion <ModuleVersion>]. - + Name : Names of one or more resources to import. ModuleName : Module names or ModuleSpecification objects of one or more modules to import. ModuleVersion : Version of module to import. If used, ModuleName must represent only one module by name. @@ -1087,7 +1084,7 @@ ModuleVersion : Version of module to import. If used, ModuleName must represent '{0}' is not a valid value for property '{1}' on class '{2}'. Please change the value to one of the following strings: {3}. - At least one of the values '{0}' is not supported or valid for property '{1}' on class '{2}'. Please specify only supported values: + At least one of the values '{0}' is not supported or valid for property '{1}' on class '{2}'. Please specify only supported values: {3}. @@ -1170,7 +1167,7 @@ ModuleVersion : Version of module to import. If used, ModuleName must represent Only one type may be specified on class members. - Error during creation of type "{0}". Error message: + Error during creation of type "{0}". Error message: {1} diff --git a/test/powershell/Language/Parser/Parser.Tests.ps1 b/test/powershell/Language/Parser/Parser.Tests.ps1 index b81175961a3..460de2c1029 100644 --- a/test/powershell/Language/Parser/Parser.Tests.ps1 +++ b/test/powershell/Language/Parser/Parser.Tests.ps1 @@ -273,16 +273,6 @@ Context "Test Unicode escape sequences." { # These tests require the file to be saved with a BOM. Unfortunately when this UTF8 file is read by # PowerShell without a BOM, the file is incorrectly interpreted as ASCII. - It 'Test that Unicode escape sequence `u2195 in string returns the ↕ character.' { - $result = ExecuteCommand '"foo`u2195abc"' - $result | should be "foo↕abc" - } - - It 'Test that Unicode escape sequence `u2195 in here string returns the ↕ character.' { - $result = ExecuteCommand ("@`"`n`n" + 'foo`u2195abc' + "`n`n`"@") - $result | should be "`nfoo↕abc`n" - } - It 'Test that the bracketed Unicode escape sequence `u{0} returns minimum char.' { $result = ExecuteCommand '"`u{0}"' [int]$result[0] | should be 0 @@ -299,38 +289,48 @@ $result | should be '©' } + It 'Test that Unicode escape sequence `u{2195} in string returns the ↕ character.' { + $result = ExecuteCommand '"foo`u{2195}abc"' + $result | should be "foo↕abc" + } + It 'Test that the bracketed Unicode escape sequence `u{1f44d} returns surrogate pair for emoji 👍 character.' { $result = ExecuteCommand '"`u{1f44d}"' $result | should be "👍" } + It 'Test that Unicode escape sequence `u{2195} in here string returns the ↕ character.' { + $result = ExecuteCommand ("@`"`n`n" + 'foo`u{2195}abc' + "`n`n`"@") + $result | should be "`nfoo↕abc`n" + } + It 'Test that Unicode escape sequence in single quoted is not processed.' { - $result = ExecuteCommand '''foo`u2195abc''' - $result | should be 'foo`u2195abc' + $result = ExecuteCommand '''foo`u{2195}abc''' + $result | should be 'foo`u{2195}abc' } It 'Test that Unicode escape sequence in single quoted here string is not processed.' { $result = ExecuteCommand @" @' -foo``u2195abc +foo``u{2195}abc '@ "@ - $result | should be "`r`nfoo``u2195abc`r`n" + $result | should be "`r`nfoo``u{2195}abc`r`n" } It "Test that two consecutive Unicode escape sequences are tokenized correctly." { - $result = ExecuteCommand '"`u007b`u007d"' + $result = ExecuteCommand '"`u{007b}`u{007d}"' $result | should be '{}' } It "Test that a Unicode escape sequence can be used in a command name." { - function xyzzy`u2195($p) {$p} - $cmd = Get-Command xyzzy`u2195 -ErrorAction SilentlyContinue + function xyzzy`u{2195}($p) {$p} + $cmd = Get-Command xyzzy`u{2195} -ErrorAction SilentlyContinue $cmd | should not BeNullOrEmpty $cmd.Name | should be 'xyzzy↕' - xyzzy`u2195 42 | should be 42 + xyzzy`u{2195} 42 | should be 42 } It "Test that a Unicode escape sequence can be used in a variable name." { diff --git a/test/powershell/Language/Parser/Parsing.Tests.ps1 b/test/powershell/Language/Parser/Parsing.Tests.ps1 index 3fcd0f8b22d..4250c7c5f56 100644 --- a/test/powershell/Language/Parser/Parsing.Tests.ps1 +++ b/test/powershell/Language/Parser/Parsing.Tests.ps1 @@ -304,14 +304,11 @@ Describe 'Hash Expression parsing' -Tags "CI" { } Describe 'Unicode escape sequence parsing' -Tag "CI" { + ShouldBeParseError '"`u{}"' InvalidUnicodeEscapeSequence 1 ShouldBeParseError '"`u{110000}"' InvalidUnicodeEscapeSequenceValue 4 # error offset is "`u{>>1<<10000}" ShouldBeParseError '"`u{1234567}"' TooManyDigitsInUnicodeEscapeSequence 10 # error offset is "`u{123456>>7<<}" - ShouldBeParseError '"`u219z"' InvalidUnicodeEscapeSequence 1 - ShouldBeParseError '"`u{219z"' MissingUnicodeEscapeSequenceTerminator 7 # error offset "`u{219>>z<<" - ShouldBeParseError '"`u{}"' EmptyUnicodeEscapeSequence 1 - ShouldBeParseError '"`u' InvalidUnicodeEscapeSequence,TerminatorExpectedAtEndOfString 1,0 - ShouldBeParseError '"`u123' InvalidUnicodeEscapeSequence,TerminatorExpectedAtEndOfString 1,0 - ShouldBeParseError '"`u219z' InvalidUnicodeEscapeSequence,TerminatorExpectedAtEndOfString 1,0 + ShouldBeParseError '"`u{219z}"' MissingUnicodeEscapeSequenceTerminator 7 # error offset "`u{219>>z<<}" + ShouldBeParseError '"`u2195}"' InvalidUnicodeEscapeSequence 1 ShouldBeParseError '"`u{' MissingUnicodeEscapeSequenceTerminator,TerminatorExpectedAtEndOfString 4,0 ShouldBeParseError '"`u{1' MissingUnicodeEscapeSequenceTerminator,TerminatorExpectedAtEndOfString 5,0 ShouldBeParseError '"`u{123456' MissingUnicodeEscapeSequenceTerminator,TerminatorExpectedAtEndOfString 10,0 From 675ba9e1dc43cf2586a5dad6bb9c8b1fb03fed33 Mon Sep 17 00:00:00 2001 From: Keith Hill Date: Fri, 23 Jun 2017 10:04:48 -0600 Subject: [PATCH 3/7] Simplify scanning of terminator. Change handling of error condition `u{12z} to indicate the sequence is not valid instead of it indicating a missing terminator. The only case now that will indicate missing terminator is `u{012345. --- .../engine/parser/tokenizer.cs | 34 +++++++------------ .../Language/Parser/Parsing.Tests.ps1 | 11 +++--- 2 files changed, 19 insertions(+), 26 deletions(-) diff --git a/src/System.Management.Automation/engine/parser/tokenizer.cs b/src/System.Management.Automation/engine/parser/tokenizer.cs index ad7482a850f..f81f9153094 100644 --- a/src/System.Management.Automation/engine/parser/tokenizer.cs +++ b/src/System.Management.Automation/engine/parser/tokenizer.cs @@ -1254,8 +1254,7 @@ private string Backtick(char c) private string ScanUnicodeEscapeSequence() { int escSeqStartIndex = _currentIndex - 2; - int maxNumberOfHexDigits = 6; - bool isTerminated = false; + const int maxNumberOfHexDigits = 6; char c = GetChar(); if (c != '{') @@ -1267,10 +1266,10 @@ private string ScanUnicodeEscapeSequence() return s_unknownUnicodeChar; } - // Scan hex chars after the Unicode escape sequence start. + // Scan the rest of the Unicode escape sequence - one to six hex digits terminated plus the closing '}'. var sb = GetStringBuilder(); int i; - for (i = 0; i < maxNumberOfHexDigits; i++) + for (i = 0; i < maxNumberOfHexDigits + 1; i++) { c = GetChar(); @@ -1285,33 +1284,26 @@ private string ScanUnicodeEscapeSequence() return s_unknownUnicodeChar; } - isTerminated = true; break; } - - if (!c.IsHexDigit()) + else if (!c.IsHexDigit()) { UngetChar(); - ReportError(_currentIndex, () => ParserStrings.MissingUnicodeEscapeSequenceTerminator); + ReportError(_currentIndex, + i < maxNumberOfHexDigits + ? (Expression>)(() => ParserStrings.InvalidUnicodeEscapeSequence) + : () => ParserStrings.MissingUnicodeEscapeSequenceTerminator); return s_unknownUnicodeChar; } - - sb.Append(c); - } - - if (!isTerminated) - { - c = GetChar(); - if (c != '}') - { + else if (i == maxNumberOfHexDigits) { UngetChar(); - ReportError(_currentIndex, - i == maxNumberOfHexDigits && c.IsHexDigit() - ? (Expression>)(() => ParserStrings.TooManyDigitsInUnicodeEscapeSequence) - : () => ParserStrings.MissingUnicodeEscapeSequenceTerminator); + + ReportError(_currentIndex, () => ParserStrings.TooManyDigitsInUnicodeEscapeSequence); return s_unknownUnicodeChar; } + + sb.Append(c); } string hexStr = GetStringAndRelease(sb); diff --git a/test/powershell/Language/Parser/Parsing.Tests.ps1 b/test/powershell/Language/Parser/Parsing.Tests.ps1 index 4250c7c5f56..0cd3bbbb475 100644 --- a/test/powershell/Language/Parser/Parsing.Tests.ps1 +++ b/test/powershell/Language/Parser/Parsing.Tests.ps1 @@ -304,13 +304,14 @@ Describe 'Hash Expression parsing' -Tags "CI" { } Describe 'Unicode escape sequence parsing' -Tag "CI" { - ShouldBeParseError '"`u{}"' InvalidUnicodeEscapeSequence 1 - ShouldBeParseError '"`u{110000}"' InvalidUnicodeEscapeSequenceValue 4 # error offset is "`u{>>1<<10000}" + ShouldBeParseError '"`u{}"' InvalidUnicodeEscapeSequence 1 # error span is >>`u{}<< + ShouldBeParseError '"`u{219z}"' InvalidUnicodeEscapeSequence 7 # error offset is "`u{219>>z<<}" + ShouldBeParseError '"`u{12345z}"' InvalidUnicodeEscapeSequence 9 # error offset is "`u{12345>>z<<}" ShouldBeParseError '"`u{1234567}"' TooManyDigitsInUnicodeEscapeSequence 10 # error offset is "`u{123456>>7<<}" - ShouldBeParseError '"`u{219z}"' MissingUnicodeEscapeSequenceTerminator 7 # error offset "`u{219>>z<<}" + ShouldBeParseError '"`u{110000}"' InvalidUnicodeEscapeSequenceValue 4 # error offset is "`u{>>1<<10000}" ShouldBeParseError '"`u2195}"' InvalidUnicodeEscapeSequence 1 - ShouldBeParseError '"`u{' MissingUnicodeEscapeSequenceTerminator,TerminatorExpectedAtEndOfString 4,0 - ShouldBeParseError '"`u{1' MissingUnicodeEscapeSequenceTerminator,TerminatorExpectedAtEndOfString 5,0 + ShouldBeParseError '"`u{' InvalidUnicodeEscapeSequence,TerminatorExpectedAtEndOfString 4,0 + ShouldBeParseError '"`u{1' InvalidUnicodeEscapeSequence,TerminatorExpectedAtEndOfString 5,0 ShouldBeParseError '"`u{123456' MissingUnicodeEscapeSequenceTerminator,TerminatorExpectedAtEndOfString 10,0 ShouldBeParseError '"`u{1234567' TooManyDigitsInUnicodeEscapeSequence,TerminatorExpectedAtEndOfString 10,0 } From 5c1b54f69651f8b887fae342a435970a6d6a5c0c Mon Sep 17 00:00:00 2001 From: Keith Hill Date: Fri, 23 Jun 2017 10:13:22 -0600 Subject: [PATCH 4/7] Promote maxNumHexDigits to static readonly int. --- .../engine/parser/tokenizer.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/System.Management.Automation/engine/parser/tokenizer.cs b/src/System.Management.Automation/engine/parser/tokenizer.cs index f81f9153094..2c85fee6e22 100644 --- a/src/System.Management.Automation/engine/parser/tokenizer.cs +++ b/src/System.Management.Automation/engine/parser/tokenizer.cs @@ -510,6 +510,7 @@ private static readonly Dictionary s_operatorTable // Unicode replacement char used to represent an unknown, unrecognized or unrepresentable character in a // Unicode escape sequence. private static readonly string s_unknownUnicodeChar = ((char)0xffdd).ToString(); + private static readonly int s_maxNumberOfUnicodeHexDigits = 6; private readonly Parser _parser; private PositionHelper _positionHelper; @@ -1254,7 +1255,6 @@ private string Backtick(char c) private string ScanUnicodeEscapeSequence() { int escSeqStartIndex = _currentIndex - 2; - const int maxNumberOfHexDigits = 6; char c = GetChar(); if (c != '{') @@ -1269,7 +1269,7 @@ private string ScanUnicodeEscapeSequence() // Scan the rest of the Unicode escape sequence - one to six hex digits terminated plus the closing '}'. var sb = GetStringBuilder(); int i; - for (i = 0; i < maxNumberOfHexDigits + 1; i++) + for (i = 0; i < s_maxNumberOfUnicodeHexDigits + 1; i++) { c = GetChar(); @@ -1291,12 +1291,12 @@ private string ScanUnicodeEscapeSequence() UngetChar(); ReportError(_currentIndex, - i < maxNumberOfHexDigits + i < s_maxNumberOfUnicodeHexDigits ? (Expression>)(() => ParserStrings.InvalidUnicodeEscapeSequence) : () => ParserStrings.MissingUnicodeEscapeSequenceTerminator); return s_unknownUnicodeChar; } - else if (i == maxNumberOfHexDigits) { + else if (i == s_maxNumberOfUnicodeHexDigits) { UngetChar(); ReportError(_currentIndex, () => ParserStrings.TooManyDigitsInUnicodeEscapeSequence); From 9419d617605f9381be17b7f08655637312b3c77a Mon Sep 17 00:00:00 2001 From: Keith Hill Date: Sun, 25 Jun 2017 11:21:04 -0600 Subject: [PATCH 5/7] Replacement char is 0xFFFD not 0xFFDD. --- src/System.Management.Automation/engine/parser/tokenizer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/System.Management.Automation/engine/parser/tokenizer.cs b/src/System.Management.Automation/engine/parser/tokenizer.cs index 2c85fee6e22..41baf8a672f 100644 --- a/src/System.Management.Automation/engine/parser/tokenizer.cs +++ b/src/System.Management.Automation/engine/parser/tokenizer.cs @@ -509,7 +509,7 @@ private static readonly Dictionary s_operatorTable // Unicode replacement char used to represent an unknown, unrecognized or unrepresentable character in a // Unicode escape sequence. - private static readonly string s_unknownUnicodeChar = ((char)0xffdd).ToString(); + private static readonly string s_unknownUnicodeChar = ((char)0xfffd).ToString(); private static readonly int s_maxNumberOfUnicodeHexDigits = 6; private readonly Parser _parser; From 04b8e45fd1108a69d5e2a0676a5c77aca40ecbc5 Mon Sep 17 00:00:00 2001 From: Keith Hill Date: Sun, 25 Jun 2017 16:45:29 -0600 Subject: [PATCH 6/7] Adopt C# approach to Unicode sequence lexing and remove IsSurrogatePair() method. The C# approach is to have the esc sequence and Unicode esc lexing return a char but also have an out char surrogateChar (lowSurrogate). This perhaps makes the Backtick() call sites a bit clearer than the use of the IsSurrogateChar() method. Also change from using s_invalidChar as 0xfffd (replacement char typically used when rendering an invalid byte in the current byte stream e.g UTF-8 reading 0x66, 0xFC, 0x72 - the 0xFC is not valid in this context and an editor might use char 0xFFFD to indicate the character was not understood. The C# parser returns 0xFFFF (Char.MaxValue) when there is an error and it needs to return something for the return value. See what you think. I can always revert this change. --- .../engine/parser/tokenizer.cs | 114 +++++++++--------- 1 file changed, 59 insertions(+), 55 deletions(-) diff --git a/src/System.Management.Automation/engine/parser/tokenizer.cs b/src/System.Management.Automation/engine/parser/tokenizer.cs index 41baf8a672f..16ca115add9 100644 --- a/src/System.Management.Automation/engine/parser/tokenizer.cs +++ b/src/System.Management.Automation/engine/parser/tokenizer.cs @@ -507,9 +507,7 @@ private static readonly Dictionary s_keywordTable private static readonly Dictionary s_operatorTable = new Dictionary(StringComparer.OrdinalIgnoreCase); - // Unicode replacement char used to represent an unknown, unrecognized or unrepresentable character in a - // Unicode escape sequence. - private static readonly string s_unknownUnicodeChar = ((char)0xfffd).ToString(); + private static readonly char s_invalidChar = char.MaxValue; private static readonly int s_maxNumberOfUnicodeHexDigits = 6; private readonly Parser _parser; @@ -1234,27 +1232,30 @@ private int SkipBlockComment(int i) return i; } - private string Backtick(char c) + private char Backtick(char c, out char surrogateCharacter) { + surrogateCharacter = s_invalidChar; + switch (c) { - case '0': return "\0"; - case 'a': return "\a"; - case 'b': return "\b"; - case 'e': return "\u001b"; - case 'f': return "\f"; - case 'n': return "\n"; - case 'r': return "\r"; - case 't': return "\t"; - case 'u': return ScanUnicodeEscapeSequence(); - case 'v': return "\v"; - default: return c.ToString(); + case '0': return '\0'; + case 'a': return '\a'; + case 'b': return '\b'; + case 'e': return '\u001b'; + case 'f': return '\f'; + case 'n': return '\n'; + case 'r': return '\r'; + case 't': return '\t'; + case 'u': return ScanUnicodeEscape(out surrogateCharacter); + case 'v': return '\v'; + default: return c; } } - private string ScanUnicodeEscapeSequence() + private char ScanUnicodeEscape(out char surrogateCharacter) { int escSeqStartIndex = _currentIndex - 2; + surrogateCharacter = s_invalidChar; char c = GetChar(); if (c != '{') @@ -1263,7 +1264,7 @@ private string ScanUnicodeEscapeSequence() IScriptExtent errorExtent = NewScriptExtent(escSeqStartIndex, _currentIndex); ReportError(errorExtent, () => ParserStrings.InvalidUnicodeEscapeSequence); - return s_unknownUnicodeChar; + return s_invalidChar; } // Scan the rest of the Unicode escape sequence - one to six hex digits terminated plus the closing '}'. @@ -1281,7 +1282,7 @@ private string ScanUnicodeEscapeSequence() // Sequence must have at least one hex char. IScriptExtent errorExtent = NewScriptExtent(escSeqStartIndex, _currentIndex); ReportError(errorExtent, () => ParserStrings.InvalidUnicodeEscapeSequence); - return s_unknownUnicodeChar; + return s_invalidChar; } break; @@ -1294,13 +1295,13 @@ private string ScanUnicodeEscapeSequence() i < s_maxNumberOfUnicodeHexDigits ? (Expression>)(() => ParserStrings.InvalidUnicodeEscapeSequence) : () => ParserStrings.MissingUnicodeEscapeSequenceTerminator); - return s_unknownUnicodeChar; + return s_invalidChar; } else if (i == s_maxNumberOfUnicodeHexDigits) { UngetChar(); ReportError(_currentIndex, () => ParserStrings.TooManyDigitsInUnicodeEscapeSequence); - return s_unknownUnicodeChar; + return s_invalidChar; } sb.Append(c); @@ -1308,41 +1309,37 @@ private string ScanUnicodeEscapeSequence() string hexStr = GetStringAndRelease(sb); - int unicodeValue = int.Parse(hexStr, NumberStyles.AllowHexSpecifier, NumberFormatInfo.InvariantInfo); + uint unicodeValue = uint.Parse(hexStr, NumberStyles.AllowHexSpecifier, NumberFormatInfo.InvariantInfo); if (unicodeValue <= Char.MaxValue) { - return ((char)unicodeValue).ToString(); + return ((char)unicodeValue); } else if (unicodeValue <= 0x10FFFF) { - return Char.ConvertFromUtf32(unicodeValue); + return GetCharsFromUtf32(unicodeValue, out surrogateCharacter); } else { // Place the error indicator under only the hex digits in the esc sequence. IScriptExtent errorExtent = NewScriptExtent(escSeqStartIndex + 3, _currentIndex - 1); ReportError(errorExtent, () => ParserStrings.InvalidUnicodeEscapeSequenceValue); - return s_unknownUnicodeChar; + return s_invalidChar; } } - private bool IsSurrogatePair(string str, ref char nonSurrogatePairChar, StringBuilder sb1 = null, StringBuilder sb2 = null) + private static char GetCharsFromUtf32(uint codepoint, out char lowSurrogate) { - Diagnostics.Assert(!string.IsNullOrEmpty(str), "Caller never calls us with an empty string"); - Diagnostics.Assert((str.Length <= 2), "Caller never calls us with a string length greater than two"); - Diagnostics.Assert((str.Length == 1) || Char.IsSurrogate(str, 0), - "Caller never calls us with a string length of two and not be a surrogate pair"); - - if (str.Length == 1) + if (codepoint < (uint)0x00010000) { - nonSurrogatePairChar = str[0]; - return false; + lowSurrogate = s_invalidChar; + return (char)codepoint; + } + else + { + Diagnostics.Assert((codepoint > 0x0000FFFF) && (codepoint <= 0x0010FFFF), "Codepoint is out of range for a surrogate pair"); + lowSurrogate = (char)((codepoint - 0x00010000) % 0x0400 + 0xDC00); + return (char)((codepoint - 0x00010000) / 0x0400 + 0xD800); } - - sb1?.Append(str); - sb2?.Append(str); - - return true; } private void ScanToEndOfCommentLine(out bool sawBeginSig, out bool matchedRequires) @@ -2128,10 +2125,11 @@ private TokenFlags ScanStringExpandable(StringBuilder sb, StringBuilder formatSb if (c1 != 0) { SkipChar(); - string str = Backtick(c1); - if (IsSurrogatePair(str, ref c, sb, formatSb)) + c = Backtick(c1, out char surrogateCharacter); + if (surrogateCharacter != s_invalidChar) { - // The character has been processed and appended to the string builders + sb.Append(c).Append(surrogateCharacter); + formatSb.Append(c).Append(surrogateCharacter); continue; } } @@ -2442,10 +2440,11 @@ private Token ScanHereStringExpandable() if (c1 != 0) { SkipChar(); - string str = Backtick(c1); - if (IsSurrogatePair(str, ref c, sb, formatSb)) + c = Backtick(c1, out char surrogateCharacter); + if (surrogateCharacter != s_invalidChar) { - // The character has been processed and appended to the string builders + sb.Append(c).Append(surrogateCharacter); + formatSb.Append(c).Append(surrogateCharacter); continue; } } @@ -2516,10 +2515,10 @@ private Token ScanVariable(bool splatted, bool inStringExpandable) UngetChar(); goto end_braced_variable_scan; } - string str = Backtick(c1); - if (IsSurrogatePair(str, ref c, sb)) + c = Backtick(c1, out char surrogateCharacter); + if (surrogateCharacter != s_invalidChar) { - // The character has been processed and appended to the string builder + sb.Append(c).Append(surrogateCharacter); continue; } break; @@ -2959,10 +2958,14 @@ private Token ScanGenericToken(char firstChar) return ScanGenericToken(sb); } - private Token ScanGenericToken(string str) + private Token ScanGenericToken(char firstChar, char surrogateCharacter) { var sb = GetStringBuilder(); - sb.Append(str); + sb.Append(firstChar); + if (surrogateCharacter != s_invalidChar) + { + sb.Append(surrogateCharacter); + } return ScanGenericToken(sb); } @@ -2992,7 +2995,6 @@ private Token ScanGenericToken(StringBuilder sb) // Make sure our token does not start with any of these characters. //Contract.Requires(Contract.ForAll("{}()@#;,|&\r\n\t ", c1 => sb[0] != c1)); - string str; List nestedTokens = new List(); var formatSb = GetStringBuilder(); formatSb.Append(sb); @@ -3007,10 +3009,11 @@ private Token ScanGenericToken(StringBuilder sb) if (c1 != 0) { SkipChar(); - str = Backtick(c1); - if (IsSurrogatePair(str, ref c, sb, formatSb)) + c = Backtick(c1, out char surrogateCharacter); + if (surrogateCharacter != s_invalidChar) { - // The character has been processed and appended to the string builders + sb.Append(c).Append(surrogateCharacter); + formatSb.Append(c).Append(surrogateCharacter); continue; } } @@ -3046,7 +3049,7 @@ private Token ScanGenericToken(StringBuilder sb) } UngetChar(); - str = GetStringAndRelease(sb); + var str = GetStringAndRelease(sb); if (nestedTokens.Count > 0) { return NewGenericExpandableToken(str, GetStringAndRelease(formatSb), nestedTokens); @@ -3928,7 +3931,8 @@ internal Token NextToken() goto again; } - return ScanGenericToken(Backtick(c1)); + c = Backtick(c1, out char surrogateCharacter); + return ScanGenericToken(c, surrogateCharacter); case '=': return CheckOperatorInCommandMode(c, TokenKind.Equals); From 32b879b2711f6113543f5735a7f752db7ffbd6ca Mon Sep 17 00:00:00 2001 From: Keith Hill Date: Mon, 26 Jun 2017 19:37:19 -0600 Subject: [PATCH 7/7] Minor perf enh to release the StringBuilder in the error conditions that return early (before the call to GetStringAndRelease). --- src/System.Management.Automation/engine/parser/tokenizer.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/System.Management.Automation/engine/parser/tokenizer.cs b/src/System.Management.Automation/engine/parser/tokenizer.cs index 16ca115add9..5a194edfd19 100644 --- a/src/System.Management.Automation/engine/parser/tokenizer.cs +++ b/src/System.Management.Automation/engine/parser/tokenizer.cs @@ -1280,6 +1280,7 @@ private char ScanUnicodeEscape(out char surrogateCharacter) if (i == 0) { // Sequence must have at least one hex char. + Release(sb); IScriptExtent errorExtent = NewScriptExtent(escSeqStartIndex, _currentIndex); ReportError(errorExtent, () => ParserStrings.InvalidUnicodeEscapeSequence); return s_invalidChar; @@ -1291,6 +1292,7 @@ private char ScanUnicodeEscape(out char surrogateCharacter) { UngetChar(); + Release(sb); ReportError(_currentIndex, i < s_maxNumberOfUnicodeHexDigits ? (Expression>)(() => ParserStrings.InvalidUnicodeEscapeSequence) @@ -1300,6 +1302,7 @@ private char ScanUnicodeEscape(out char surrogateCharacter) else if (i == s_maxNumberOfUnicodeHexDigits) { UngetChar(); + Release(sb); ReportError(_currentIndex, () => ParserStrings.TooManyDigitsInUnicodeEscapeSequence); return s_invalidChar; }