From 38c08895735234eeb955f1726adecb20614fc679 Mon Sep 17 00:00:00 2001 From: Kirk Munro Date: Mon, 18 Feb 2019 23:50:52 -0400 Subject: [PATCH 01/27] support continuance with pipe at beginning of line --- .../engine/parser/Parser.cs | 23 +++++++- .../engine/parser/tokenizer.cs | 57 ++++++++++++++++--- 2 files changed, 69 insertions(+), 11 deletions(-) diff --git a/src/System.Management.Automation/engine/parser/Parser.cs b/src/System.Management.Automation/engine/parser/Parser.cs index 273c4d561a0..f604ee6d5b4 100644 --- a/src/System.Management.Automation/engine/parser/Parser.cs +++ b/src/System.Management.Automation/engine/parser/Parser.cs @@ -351,7 +351,7 @@ private void SkipNewlines() if (_ungotToken == null || _ungotToken.Kind == TokenKind.NewLine) { _ungotToken = null; - _tokenizer.SkipNewlines(false, false); + _tokenizer.SkipNewlines(); } } @@ -362,7 +362,7 @@ private void V3SkipNewlines() if (_ungotToken == null || _ungotToken.Kind == TokenKind.NewLine) { _ungotToken = null; - _tokenizer.SkipNewlines(false, true); + _tokenizer.SkipNewlines(NewlineSkipOption.V3); } } @@ -371,7 +371,16 @@ private void SkipNewlinesAndSemicolons() if (_ungotToken == null || _ungotToken.Kind == TokenKind.NewLine || _ungotToken.Kind == TokenKind.Semi) { _ungotToken = null; - _tokenizer.SkipNewlines(true, false); + _tokenizer.SkipNewlines(NewlineSkipOption.SkipSemis); + } + } + + private void SkipNewlinesBeforePipe() + { + if (_ungotToken == null || _ungotToken.Kind == TokenKind.NewLine) + { + _ungotToken = null; + _tokenizer.SkipNewlines(NewlineSkipOption.BeforePipe); } } @@ -5700,6 +5709,8 @@ private PipelineBaseAst PipelineRule() // G pipeline-tail: // G '|' new-lines:opt command // G '|' new-lines:opt command pipeline-tail + // G new-lines:opt '|' command + // G new-lines:opt '|' command pipeline-tail var pipelineElements = new List(); IScriptExtent startExtent = null; @@ -5817,6 +5828,12 @@ private PipelineBaseAst PipelineRule() } pipeToken = PeekToken(); + // Skip newlines before pipe tokens to support pipe tokens at the start of the next line of script + if (pipeToken.Kind == TokenKind.NewLine) + { + SkipNewlinesBeforePipe(); + pipeToken = PeekToken(); + } switch (pipeToken.Kind) { diff --git a/src/System.Management.Automation/engine/parser/tokenizer.cs b/src/System.Management.Automation/engine/parser/tokenizer.cs index aab4571a9c7..5d969264d12 100644 --- a/src/System.Management.Automation/engine/parser/tokenizer.cs +++ b/src/System.Management.Automation/engine/parser/tokenizer.cs @@ -559,6 +559,29 @@ internal class TokenizerState internal List TokenList; } + /// + /// Defines the option to use when skipping newlines. + /// + internal enum NewlineSkipOption + { + /// + /// Simply skip newlines. + /// + None, + /// + /// Skip newlines and semi-colons. + /// + SkipSemis, + /// + /// Skip newlines using the old v3 method. + /// + V3, + /// + /// Skip newlines that are before a pipe. + /// + BeforePipe + } + [DebuggerDisplay("Mode = {Mode}; Script = {_script}")] internal class Tokenizer { @@ -848,10 +871,13 @@ internal static bool IsKeyword(string str) return false; } - internal void SkipNewlines(bool skipSemis, bool v3) + internal void SkipNewlines(NewlineSkipOption skipOption = NewlineSkipOption.None) { - // We normally don't create any tokens here, but the V2 tokenizer api returns newline tokens, - // so we create them when asked to create them. + // We normally don't create any tokens here, but the V2 tokenizer api returns newline tokens, + // so we create them when asked to create them. + + int startIndex = _currentIndex; + var tokenQueue = new Queue(); again: char c = GetChar(); @@ -868,23 +894,23 @@ internal void SkipNewlines(bool skipSemis, bool v3) case '\r': case '\n': - if (v3) _parser.NoteV3FeatureUsed(); + if (skipOption == NewlineSkipOption.V3) _parser.NoteV3FeatureUsed(); if (TokenList != null) { _tokenStart = _currentIndex - 1; ScanNewline(c); - NewToken(TokenKind.NewLine); + tokenQueue.Enqueue(new Token(CurrentExtent(), TokenKind.NewLine, TokenFlags.None)); } goto again; case ';': - if (skipSemis) + if (skipOption == NewlineSkipOption.SkipSemis) { if (TokenList != null) { _tokenStart = _currentIndex - 1; - NewToken(TokenKind.Semi); + tokenQueue.Enqueue(new Token(CurrentExtent(), TokenKind.Semi, TokenFlags.None)); } goto again; @@ -914,7 +940,7 @@ internal void SkipNewlines(bool skipSemis, bool v3) { _tokenStart = _currentIndex - 2; ScanNewline(c1); - NewToken(TokenKind.LineContinuation); + tokenQueue.Enqueue(new Token(CurrentExtent(), TokenKind.LineContinuation, TokenFlags.None)); goto again; } @@ -934,9 +960,24 @@ internal void SkipNewlines(bool skipSemis, bool v3) goto again; } + if (skipOption == NewlineSkipOption.BeforePipe && c != '|') + { + tokenQueue.Clear(); + + while (_currentIndex > startIndex) + { + UngetChar(); + } + } + break; } + while (tokenQueue.Count > 0) + { + SaveToken(tokenQueue.Dequeue()); + } + UngetChar(); } From f71a592d558a9ea60d4cc5aec8d017187d9bf891 Mon Sep 17 00:00:00 2001 From: Kirk Munro Date: Wed, 20 Feb 2019 16:38:51 -0400 Subject: [PATCH 02/27] token management logic changed to fix highlighting --- .../engine/parser/tokenizer.cs | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/System.Management.Automation/engine/parser/tokenizer.cs b/src/System.Management.Automation/engine/parser/tokenizer.cs index 5d969264d12..ccdf6240d3e 100644 --- a/src/System.Management.Automation/engine/parser/tokenizer.cs +++ b/src/System.Management.Automation/engine/parser/tokenizer.cs @@ -877,7 +877,7 @@ internal void SkipNewlines(NewlineSkipOption skipOption = NewlineSkipOption.None // so we create them when asked to create them. int startIndex = _currentIndex; - var tokenQueue = new Queue(); + var tokenCount = TokenList != null ? TokenList.Count : 0; again: char c = GetChar(); @@ -899,7 +899,7 @@ internal void SkipNewlines(NewlineSkipOption skipOption = NewlineSkipOption.None { _tokenStart = _currentIndex - 1; ScanNewline(c); - tokenQueue.Enqueue(new Token(CurrentExtent(), TokenKind.NewLine, TokenFlags.None)); + NewToken(TokenKind.NewLine); } goto again; @@ -910,7 +910,7 @@ internal void SkipNewlines(NewlineSkipOption skipOption = NewlineSkipOption.None if (TokenList != null) { _tokenStart = _currentIndex - 1; - tokenQueue.Enqueue(new Token(CurrentExtent(), TokenKind.Semi, TokenFlags.None)); + NewToken(TokenKind.Semi); } goto again; @@ -940,7 +940,7 @@ internal void SkipNewlines(NewlineSkipOption skipOption = NewlineSkipOption.None { _tokenStart = _currentIndex - 2; ScanNewline(c1); - tokenQueue.Enqueue(new Token(CurrentExtent(), TokenKind.LineContinuation, TokenFlags.None)); + NewToken(TokenKind.LineContinuation); goto again; } @@ -960,9 +960,25 @@ internal void SkipNewlines(NewlineSkipOption skipOption = NewlineSkipOption.None goto again; } + // If the first non-newline, non-whitespace, non-comment, non-backtick, non-semi-colon character + // following the newline is not a pipe, remove the tokens that were added while tokenizing and unget + // the characters that were retrieved. This allows us to "lookahead" to see if there is a pipe and + // only apply line continuance automatically when one is found. if (skipOption == NewlineSkipOption.BeforePipe && c != '|') { - tokenQueue.Clear(); + var newTokenCount = TokenList != null ? TokenList.Count : 0; + if (newTokenCount > tokenCount) + { + if (tokenCount == 0) + { + TokenList.Clear(); + TokenList = null; + } + else + { + TokenList.RemoveRange(tokenCount - 1, newTokenCount - tokenCount + 1); + } + } while (_currentIndex > startIndex) { @@ -973,11 +989,6 @@ internal void SkipNewlines(NewlineSkipOption skipOption = NewlineSkipOption.None break; } - while (tokenQueue.Count > 0) - { - SaveToken(tokenQueue.Dequeue()); - } - UngetChar(); } From 10c47800df9ad52940b64348b6148d8e9eecf691 Mon Sep 17 00:00:00 2001 From: Kirk Munro Date: Wed, 20 Feb 2019 16:50:38 -0400 Subject: [PATCH 03/27] cleaned up new comments --- src/System.Management.Automation/engine/parser/Parser.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/System.Management.Automation/engine/parser/Parser.cs b/src/System.Management.Automation/engine/parser/Parser.cs index f604ee6d5b4..3dcccecb50f 100644 --- a/src/System.Management.Automation/engine/parser/Parser.cs +++ b/src/System.Management.Automation/engine/parser/Parser.cs @@ -5707,10 +5707,7 @@ private PipelineBaseAst PipelineRule() // G expression assignment-operator statement // G // G pipeline-tail: - // G '|' new-lines:opt command - // G '|' new-lines:opt command pipeline-tail - // G new-lines:opt '|' command - // G new-lines:opt '|' command pipeline-tail + // G new-lines:opt '|' new-lines:opt command pipeline-tail:opt var pipelineElements = new List(); IScriptExtent startExtent = null; @@ -5828,7 +5825,9 @@ private PipelineBaseAst PipelineRule() } pipeToken = PeekToken(); - // Skip newlines before pipe tokens to support pipe tokens at the start of the next line of script + + // Skip newlines before pipe tokens to support (pipe)line continuance when pipe + // tokens start the next line of script if (pipeToken.Kind == TokenKind.NewLine) { SkipNewlinesBeforePipe(); From 7092a36047c5291521a55e2c55f9ab127b48299f Mon Sep 17 00:00:00 2001 From: Kirk Munro Date: Thu, 21 Feb 2019 07:45:43 -0400 Subject: [PATCH 04/27] added pester tests --- .../Language/Parser/LineContinuance.Tests.ps1 | 247 ++++++++++++++++++ .../Language/Parser/Parsing.Tests.ps1 | 1 + 2 files changed, 248 insertions(+) create mode 100644 test/powershell/Language/Parser/LineContinuance.Tests.ps1 diff --git a/test/powershell/Language/Parser/LineContinuance.Tests.ps1 b/test/powershell/Language/Parser/LineContinuance.Tests.ps1 new file mode 100644 index 00000000000..e7c52b940f8 --- /dev/null +++ b/test/powershell/Language/Parser/LineContinuance.Tests.ps1 @@ -0,0 +1,247 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +Describe 'Line Continuance' -Tags 'CI' { + BeforeAll { + function ExecuteCommand { + param ([string]$command) + [powershell]::Create().AddScript($command).Invoke() + } + } + + Context 'Lines ending with a backtick that parse and execute without error' { + It 'Lines ending with a single backtick' { + $script = @' +'Hello' + ` + ' world' +'@ + ExecuteCommand $script | Should -Be 'Hello world' + } + + It 'Lines ending with a single backtick followed by whitespace' { + $script = @' +# The next line ends with trailing whitespace +'Hello' + ` + ' world' +'@ + ExecuteCommand $script | Should -Be 'Hello world' + } + + It 'Lines ending with a single backtick followed by whitespace and a comment' { + $script = @' +'Hello' + ` # You can place comments after whitespace following backticks + ' world' +'@ + ExecuteCommand $script | Should -Be 'Hello world' + } + + It 'Lines ending with a single backtick followed by a comment line and then the continued line' { + $script = @' +'Hello' + ` +# You can place comments in the middle of a continued line + ' world' +'@ + ExecuteCommand $script | Should -Be 'Hello world' + } + } + + Context 'Lines ending with a backtick that do not parse' { + It 'Lines ending with a single backtick followed immediately by a comment' { + $script = @' +'Hello' + `# This is not a valid comment + ' world' +'@ + $err = { ExecuteCommand $script } | Should -Throw -ErrorId 'ParseException' -PassThru + $err.Exception.InnerException.ErrorRecord.FullyQualifiedErrorId | Should -BeExactly 'ExpectedValueExpression' + } + + It 'Lines ending with two backticks' { + $script = @' +'Hello' + `` + ' world' +'@ + $err = { ExecuteCommand $script } | Should -Throw -ErrorId 'ParseException' -PassThru + $err.Exception.InnerException.ErrorRecord.FullyQualifiedErrorId | Should -BeExactly 'ExpectedValueExpression' + } + } + + Context 'Lines ending with a pipe that parse and execute without error' { + It 'Lines ending with a pipe' { + $script = @' +'Hello' | + ForEach-Object {"$_ world"} +'@ + ExecuteCommand $script | Should -Be 'Hello world' + } + + It 'Lines ending with a pipe followed by whitespace' { + $script = @' +# The next line ends with trailing whitespace +'Hello' | + ForEach-Object {"$_ world"} +'@ + ExecuteCommand $script | Should -Be 'Hello world' + } + + It 'Lines ending with a pipe followed by a comment' { + $script = @' +'Hello' |# You can place comments after whitespace following pipes + ForEach-Object {"$_ world"} +'@ + ExecuteCommand $script | Should -Be 'Hello world' + } + + It 'Lines ending with a pipe followed by a comment line and then the continued command' { + $script = @' +'Hello' | +# You can place comments in the middle of a continued pipeline + ForEach-Object {"$_ world"} +'@ + ExecuteCommand $script | Should -Be 'Hello world' + } + } + + Context 'Lines ending with a pipe that do not parse' { + It 'Lines ending with a single pipe followed by an empty line' { + $script = @' +'Hello' | + +'@ + $err = { ExecuteCommand $script } | Should -Throw -ErrorId 'ParseException' -PassThru + $err.Exception.InnerException.ErrorRecord.FullyQualifiedErrorId | Should -BeExactly 'EmptyPipeElement' + } + + It 'Lines ending with two pipes' { + $script = @' +'Hello' || + +'@ + $err = { ExecuteCommand $script } | Should -Throw -ErrorId 'ParseException' -PassThru + $err.Exception.InnerException.ErrorRecord.FullyQualifiedErrorId | Should -BeExactly 'ExpectedValueExpression' + } + } + + Context 'Parsing and executing without error while using a pipe at the beginning of a line to continue the previous line' { + It 'Line continuance using a pipe at the start of a subsequent line' { + $script = @' +'Hello' + | ForEach-Object {"$_ world"} +'@ + ExecuteCommand $script | Should -Be 'Hello world' + } + + It 'Longer line continuance using pipes at the start of subsequent lines' { + $script = @' +1..10 + | ForEach-Object {($_ -shl 1) + $_} + | Where-Object {$_ % 2 -eq 0} + | Sort-Object -Descending +'@ + ExecuteCommand $script | Should -Be @(30, 24, 18, 12, 6) + } + + It 'Line continuance using a comment line followed by a pipe at the start of a subsequent line' { + $script = @' +'Hello' + # You can place comments before continued pipelines + | ForEach-Object {"$_ world"} +'@ + ExecuteCommand $script | Should -Be 'Hello world' + } + + It 'Longer line continuance using pipes at the start of subsequent lines (with comments)' { + $script = @' +1..10 + # Times three + | ForEach-Object {($_ -shl 1) + $_} + # Even only + | Where-Object {$_ % 2 -eq 0} + # Reverse order + | Sort-Object -Descending +'@ + ExecuteCommand $script | Should -Be @(30, 24, 18, 12, 6) + } + + It 'Line continuance using a pipe at the start of a subsequent line after multiple blank lines' { + $script = @' +'Hello' + + + | ForEach-Object {"$_ world"} +'@ + ExecuteCommand $script | Should -Be 'Hello world' + } + + It 'Line continuance using a pipe on a line by itself' { + $script = @' +'Hello' + | + ForEach-Object {"$_ world"} +'@ + ExecuteCommand $script | Should -Be 'Hello world' + } + + + It 'Longer line continuance using pipes on lines by themselves' { + $script = @' +1..10 + | + ForEach-Object {($_ -shl 1) + $_} + | + Where-Object {$_ % 2 -eq 0} + | + Sort-Object -Descending +'@ + ExecuteCommand $script | Should -Be @(30, 24, 18, 12, 6) + } + + It 'Line continuance using a pipe on a line by itself (with comments)' { + $script = @' +'Hello' + | + # You can place comments before continued pipelines + ForEach-Object {"$_ world"} +'@ + ExecuteCommand $script | Should -Be 'Hello world' + } + + It 'Longer line continuance using pipes on lines by themselves (with comments)' { + $script = @' +1..10 + | + # Times three + ForEach-Object {($_ -shl 1) + $_} + | + # Even only + Where-Object {$_ % 2 -eq 0} + | + # Reverse order + Sort-Object -Descending +'@ + ExecuteCommand $script | Should -Be @(30, 24, 18, 12, 6) + } + } + + Context 'Lines starting with a pipe that do not parse' { + It 'Lines starting with a single pipe with nothing after it' { + $script = @' +'Hello' + | # Nothing to see here, move along + +'@ + $err = { ExecuteCommand $script } | Should -Throw -ErrorId 'ParseException' -PassThru + $err.Exception.InnerException.ErrorRecord.FullyQualifiedErrorId | Should -BeExactly 'EmptyPipeElement' + } + + It 'Lines starting with a single pipe with blank lines after it' { + $script = @' +'Hello' + | + + +'@ + $err = { ExecuteCommand $script } | Should -Throw -ErrorId 'ParseException' -PassThru + $err.Exception.InnerException.ErrorRecord.FullyQualifiedErrorId | Should -BeExactly 'EmptyPipeElement' + } + } +} diff --git a/test/powershell/Language/Parser/Parsing.Tests.ps1 b/test/powershell/Language/Parser/Parsing.Tests.ps1 index 9737a090100..be4a07874d4 100644 --- a/test/powershell/Language/Parser/Parsing.Tests.ps1 +++ b/test/powershell/Language/Parser/Parsing.Tests.ps1 @@ -270,6 +270,7 @@ Describe 'splatting parsing' -Tags "CI" { } Describe 'Pipes parsing' -Tags "CI" { + ShouldBeParseError '|gps' EmptyPipeElement 0 ShouldBeParseError 'gps|' EmptyPipeElement 4 ShouldBeParseError '1|1' ExpressionsMustBeFirstInPipeline 2 ShouldBeParseError '$a=' ExpectedValueExpression 3 From 51c73e274cd97d198fed339eb5cd8c5a788f1559 Mon Sep 17 00:00:00 2001 From: Kirk Munro Date: Thu, 21 Feb 2019 13:29:24 -0400 Subject: [PATCH 05/27] CodeFactor changes --- src/System.Management.Automation/engine/parser/tokenizer.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/System.Management.Automation/engine/parser/tokenizer.cs b/src/System.Management.Automation/engine/parser/tokenizer.cs index ccdf6240d3e..d500ab6e266 100644 --- a/src/System.Management.Automation/engine/parser/tokenizer.cs +++ b/src/System.Management.Automation/engine/parser/tokenizer.cs @@ -568,14 +568,17 @@ internal enum NewlineSkipOption /// Simply skip newlines. /// None, + /// /// Skip newlines and semi-colons. /// SkipSemis, + /// /// Skip newlines using the old v3 method. /// V3, + /// /// Skip newlines that are before a pipe. /// @@ -875,7 +878,6 @@ internal void SkipNewlines(NewlineSkipOption skipOption = NewlineSkipOption.None { // We normally don't create any tokens here, but the V2 tokenizer api returns newline tokens, // so we create them when asked to create them. - int startIndex = _currentIndex; var tokenCount = TokenList != null ? TokenList.Count : 0; @@ -894,7 +896,7 @@ internal void SkipNewlines(NewlineSkipOption skipOption = NewlineSkipOption.None case '\r': case '\n': - if (skipOption == NewlineSkipOption.V3) _parser.NoteV3FeatureUsed(); + if (skipOption == NewlineSkipOption.V3) { _parser.NoteV3FeatureUsed(); } if (TokenList != null) { _tokenStart = _currentIndex - 1; From 7e0a0e45428aaa75f3a724bdfe4ee4387030c5c4 Mon Sep 17 00:00:00 2001 From: Kirk Munro Date: Thu, 21 Feb 2019 13:43:15 -0400 Subject: [PATCH 06/27] Removed legacy V3 code that did nothing --- .../engine/parser/Parser.cs | 30 ++++--------------- .../engine/parser/tokenizer.cs | 6 ---- 2 files changed, 5 insertions(+), 31 deletions(-) diff --git a/src/System.Management.Automation/engine/parser/Parser.cs b/src/System.Management.Automation/engine/parser/Parser.cs index 3dcccecb50f..203627e20f2 100644 --- a/src/System.Management.Automation/engine/parser/Parser.cs +++ b/src/System.Management.Automation/engine/parser/Parser.cs @@ -42,7 +42,6 @@ public sealed class Parser private bool _inConfiguration; private ParseMode _parseMode; - // private bool _v3FeatureUsed; internal string _fileName; internal bool ProduceV2Tokens { get; set; } @@ -340,8 +339,6 @@ internal void SetPreviousFirstLastToken(ExecutionContext context) } } - // public bool V3FeatureUsed { get { return _v3FeatureUsed; } } - internal List ErrorList { get; } #region Utilities @@ -355,17 +352,6 @@ private void SkipNewlines() } } - // Same as SkipNewlines, but remembers is used when we skip lines differently in - // V3. - private void V3SkipNewlines() - { - if (_ungotToken == null || _ungotToken.Kind == TokenKind.NewLine) - { - _ungotToken = null; - _tokenizer.SkipNewlines(NewlineSkipOption.V3); - } - } - private void SkipNewlinesAndSemicolons() { if (_ungotToken == null || _ungotToken.Kind == TokenKind.NewLine || _ungotToken.Kind == TokenKind.Semi) @@ -576,11 +562,6 @@ private static bool IsSpecificParameter(Token token, string parameter) return parameter.StartsWith(paramToken.ParameterName, StringComparison.OrdinalIgnoreCase); } - internal void NoteV3FeatureUsed() - { - // _v3FeatureUsed = true; - } - internal void RequireStatementTerminator() { var terminatorToken = PeekToken(); @@ -1095,7 +1076,7 @@ private AttributeBaseAst AttributeRule() return null; } - V3SkipNewlines(); + SkipNewlines(); Token firstTypeNameToken; ITypeName typeName = TypeNameRule(allowAssemblyQualifiedNames: true, firstTypeNameToken: out firstTypeNameToken); @@ -1238,7 +1219,6 @@ private void AttributeArgumentsRule(ICollection positionalArgumen // and record that it was defaulted for better error messages. expr = new ConstantExpressionAst(name.Extent, true); expressionOmitted = true; - NoteV3FeatureUsed(); } } else @@ -1370,7 +1350,7 @@ private ITypeName FinishTypeNameRule(Token typeName, bool unBracketedGenericArg // Array or generic SkipToken(); - V3SkipNewlines(); + SkipNewlines(); token = NextToken(); switch (token.Kind) { @@ -1474,14 +1454,14 @@ private ITypeName GenericTypeArgumentsRule(Token genericTypeName, Token firstTok Token token; while (true) { - V3SkipNewlines(); + SkipNewlines(); commaOrRBracketToken = NextToken(); if (commaOrRBracketToken.Kind != TokenKind.Comma) { break; } - V3SkipNewlines(); + SkipNewlines(); token = PeekToken(); if (token.Kind == TokenKind.Identifier || token.Kind == TokenKind.LBracket) @@ -6906,7 +6886,7 @@ private ExpressionAst CheckPostPrimaryExpressionOperators(Token token, Expressio while (token != null) { // To support fluent style programming, allow newlines after the member access operator. - V3SkipNewlines(); + SkipNewlines(); if (token.Kind == TokenKind.Dot || token.Kind == TokenKind.ColonColon) { diff --git a/src/System.Management.Automation/engine/parser/tokenizer.cs b/src/System.Management.Automation/engine/parser/tokenizer.cs index d500ab6e266..4f42fbb857a 100644 --- a/src/System.Management.Automation/engine/parser/tokenizer.cs +++ b/src/System.Management.Automation/engine/parser/tokenizer.cs @@ -574,11 +574,6 @@ internal enum NewlineSkipOption /// SkipSemis, - /// - /// Skip newlines using the old v3 method. - /// - V3, - /// /// Skip newlines that are before a pipe. /// @@ -896,7 +891,6 @@ internal void SkipNewlines(NewlineSkipOption skipOption = NewlineSkipOption.None case '\r': case '\n': - if (skipOption == NewlineSkipOption.V3) { _parser.NoteV3FeatureUsed(); } if (TokenList != null) { _tokenStart = _currentIndex - 1; From ad12433f43edee02090efb1b4ef17ac97c61e9fd Mon Sep 17 00:00:00 2001 From: Kirk Munro Date: Thu, 21 Feb 2019 14:05:09 -0400 Subject: [PATCH 07/27] updated pester tests --- .../Language/Parser/LineContinuance.Tests.ps1 | 13 ++++++++----- test/powershell/Language/Parser/Parsing.Tests.ps1 | 1 + 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/test/powershell/Language/Parser/LineContinuance.Tests.ps1 b/test/powershell/Language/Parser/LineContinuance.Tests.ps1 index e7c52b940f8..b08c15fd9d6 100644 --- a/test/powershell/Language/Parser/LineContinuance.Tests.ps1 +++ b/test/powershell/Language/Parser/LineContinuance.Tests.ps1 @@ -20,8 +20,10 @@ Describe 'Line Continuance' -Tags 'CI' { It 'Lines ending with a single backtick followed by whitespace' { $script = @' -# The next line ends with trailing whitespace -'Hello' + ` +# The first line of this command ends with trailing whitespace +'Hello' + ` +'@ + ' ' + @' + ' world' '@ ExecuteCommand $script | Should -Be 'Hello world' @@ -111,13 +113,14 @@ Describe 'Line Continuance' -Tags 'CI' { $err.Exception.InnerException.ErrorRecord.FullyQualifiedErrorId | Should -BeExactly 'EmptyPipeElement' } - It 'Lines ending with two pipes' { + It 'Lines ending with a single pipe followed by a line that starts with a pipe' { $script = @' -'Hello' || +'Hello' | + | '@ $err = { ExecuteCommand $script } | Should -Throw -ErrorId 'ParseException' -PassThru - $err.Exception.InnerException.ErrorRecord.FullyQualifiedErrorId | Should -BeExactly 'ExpectedValueExpression' + $err.Exception.InnerException.ErrorRecord.FullyQualifiedErrorId | Should -BeExactly 'EmptyPipeElement' } } diff --git a/test/powershell/Language/Parser/Parsing.Tests.ps1 b/test/powershell/Language/Parser/Parsing.Tests.ps1 index be4a07874d4..afb15081124 100644 --- a/test/powershell/Language/Parser/Parsing.Tests.ps1 +++ b/test/powershell/Language/Parser/Parsing.Tests.ps1 @@ -272,6 +272,7 @@ Describe 'splatting parsing' -Tags "CI" { Describe 'Pipes parsing' -Tags "CI" { ShouldBeParseError '|gps' EmptyPipeElement 0 ShouldBeParseError 'gps|' EmptyPipeElement 4 + ShouldBeParseError 'gps| |foreach name' EmptyPipeElement 4 ShouldBeParseError '1|1' ExpressionsMustBeFirstInPipeline 2 ShouldBeParseError '$a=' ExpectedValueExpression 3 } From ec4761f182932f98be7e24b4be07d418dd9f8586 Mon Sep 17 00:00:00 2001 From: Kirk Munro Date: Sun, 3 Mar 2019 21:01:53 -0400 Subject: [PATCH 08/27] made whitespace at end of line intentional --- test/powershell/Language/Parser/LineContinuance.Tests.ps1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/powershell/Language/Parser/LineContinuance.Tests.ps1 b/test/powershell/Language/Parser/LineContinuance.Tests.ps1 index b08c15fd9d6..bbebd2b73c7 100644 --- a/test/powershell/Language/Parser/LineContinuance.Tests.ps1 +++ b/test/powershell/Language/Parser/LineContinuance.Tests.ps1 @@ -79,7 +79,9 @@ Describe 'Line Continuance' -Tags 'CI' { It 'Lines ending with a pipe followed by whitespace' { $script = @' # The next line ends with trailing whitespace -'Hello' | +'Hello' | +'@ + ' ' + @' + ForEach-Object {"$_ world"} '@ ExecuteCommand $script | Should -Be 'Hello world' From f3d2f49d3abb99b0dfba02e492b3e50377641cb2 Mon Sep 17 00:00:00 2001 From: Kirk Munro Date: Wed, 3 Apr 2019 16:13:32 -0300 Subject: [PATCH 09/27] refactored for performance and to fix syntax highlighting --- .../engine/parser/Parser.cs | 41 ++-- .../engine/parser/tokenizer.cs | 186 +++++++++++------- 2 files changed, 130 insertions(+), 97 deletions(-) diff --git a/src/System.Management.Automation/engine/parser/Parser.cs b/src/System.Management.Automation/engine/parser/Parser.cs index c0d79d1d8d7..66a7f958ab1 100644 --- a/src/System.Management.Automation/engine/parser/Parser.cs +++ b/src/System.Management.Automation/engine/parser/Parser.cs @@ -348,7 +348,7 @@ private void SkipNewlines() if (_ungotToken == null || _ungotToken.Kind == TokenKind.NewLine) { _ungotToken = null; - _tokenizer.SkipNewlines(); + _tokenizer.SkipNewlines(false); } } @@ -357,16 +357,7 @@ private void SkipNewlinesAndSemicolons() if (_ungotToken == null || _ungotToken.Kind == TokenKind.NewLine || _ungotToken.Kind == TokenKind.Semi) { _ungotToken = null; - _tokenizer.SkipNewlines(NewlineSkipOption.SkipSemis); - } - } - - private void SkipNewlinesBeforePipe() - { - if (_ungotToken == null || _ungotToken.Kind == TokenKind.NewLine) - { - _ungotToken = null; - _tokenizer.SkipNewlines(NewlineSkipOption.BeforePipe); + _tokenizer.SkipNewlines(true); } } @@ -426,7 +417,7 @@ private void SyncOnError(bool consumeClosingToken, params TokenKind[] syncTokens break; case TokenKind.EndOfInput: - // Never consume , but return it so caller + // Never consume , but return it to caller UngetToken(token); return; } @@ -5703,7 +5694,7 @@ private PipelineBaseAst PipelineRule() var pipelineElements = new List(); IScriptExtent startExtent = null; - Token pipeToken = null; + Token nextToken = null; bool scanning = true; bool background = false; while (scanning) @@ -5809,28 +5800,28 @@ private PipelineBaseAst PipelineRule() // If the first pipe element is null, the position points to the pipe (ideally it would // point before, but the pipe could be the first character), otherwise the empty element // is after the pipe character. - IScriptExtent errorPosition = pipeToken != null ? After(pipeToken) : PeekToken().Extent; + IScriptExtent errorPosition = nextToken != null ? After(nextToken) : PeekToken().Extent; ReportIncompleteInput(errorPosition, nameof(ParserStrings.EmptyPipeElement), ParserStrings.EmptyPipeElement); } - pipeToken = PeekToken(); + nextToken = PeekToken(); // Skip newlines before pipe tokens to support (pipe)line continuance when pipe // tokens start the next line of script - if (pipeToken.Kind == TokenKind.NewLine) + if (nextToken.Kind == TokenKind.NewLine && _tokenizer.IsPipeContinuance(nextToken.Extent)) { - SkipNewlinesBeforePipe(); - pipeToken = PeekToken(); + SkipNewlines(); + nextToken = PeekToken(); } - switch (pipeToken.Kind) + switch (nextToken.Kind) { case TokenKind.Semi: - case TokenKind.NewLine: case TokenKind.RParen: case TokenKind.RCurly: + case TokenKind.NewLine: case TokenKind.EndOfInput: scanning = false; continue; @@ -5845,7 +5836,7 @@ private PipelineBaseAst PipelineRule() if (PeekToken().Kind == TokenKind.EndOfInput) { scanning = false; - ReportIncompleteInput(After(pipeToken), + ReportIncompleteInput(After(nextToken), nameof(ParserStrings.EmptyPipeElement), ParserStrings.EmptyPipeElement); } @@ -5856,10 +5847,10 @@ private PipelineBaseAst PipelineRule() // Parse in a manner similar to a pipe, but issue an error (for now, but should implement this for V3.) SkipToken(); SkipNewlines(); - ReportError(pipeToken.Extent, + ReportError(nextToken.Extent, nameof(ParserStrings.InvalidEndOfLine), ParserStrings.InvalidEndOfLine, - pipeToken.Text); + nextToken.Text); if (PeekToken().Kind == TokenKind.EndOfInput) { scanning = false; @@ -5870,10 +5861,10 @@ private PipelineBaseAst PipelineRule() default: // ErrorRecovery: don't eat the token, assume it belongs to something else. - ReportError(pipeToken.Extent, + ReportError(nextToken.Extent, nameof(ParserStrings.UnexpectedToken), ParserStrings.UnexpectedToken, - pipeToken.Text); + nextToken.Text); scanning = false; break; } diff --git a/src/System.Management.Automation/engine/parser/tokenizer.cs b/src/System.Management.Automation/engine/parser/tokenizer.cs index 4f42fbb857a..f992692a8ce 100644 --- a/src/System.Management.Automation/engine/parser/tokenizer.cs +++ b/src/System.Management.Automation/engine/parser/tokenizer.cs @@ -559,27 +559,6 @@ internal class TokenizerState internal List TokenList; } - /// - /// Defines the option to use when skipping newlines. - /// - internal enum NewlineSkipOption - { - /// - /// Simply skip newlines. - /// - None, - - /// - /// Skip newlines and semi-colons. - /// - SkipSemis, - - /// - /// Skip newlines that are before a pipe. - /// - BeforePipe - } - [DebuggerDisplay("Mode = {Mode}; Script = {_script}")] internal class Tokenizer { @@ -869,12 +848,10 @@ internal static bool IsKeyword(string str) return false; } - internal void SkipNewlines(NewlineSkipOption skipOption = NewlineSkipOption.None) + internal void SkipNewlines(bool skipSemis) { // We normally don't create any tokens here, but the V2 tokenizer api returns newline tokens, // so we create them when asked to create them. - int startIndex = _currentIndex; - var tokenCount = TokenList != null ? TokenList.Count : 0; again: char c = GetChar(); @@ -891,24 +868,13 @@ internal void SkipNewlines(NewlineSkipOption skipOption = NewlineSkipOption.None case '\r': case '\n': - if (TokenList != null) - { - _tokenStart = _currentIndex - 1; - ScanNewline(c); - NewToken(TokenKind.NewLine); - } - + ScanNewline(c); goto again; case ';': - if (skipOption == NewlineSkipOption.SkipSemis) + if (skipSemis) { - if (TokenList != null) - { - _tokenStart = _currentIndex - 1; - NewToken(TokenKind.Semi); - } - + ScanSemicolon(); goto again; } @@ -934,9 +900,7 @@ internal void SkipNewlines(NewlineSkipOption skipOption = NewlineSkipOption.None char c1 = GetChar(); if (c1 == '\n' || c1 == '\r') { - _tokenStart = _currentIndex - 2; - ScanNewline(c1); - NewToken(TokenKind.LineContinuation); + ScanLineContinuation(c1); goto again; } @@ -956,32 +920,6 @@ internal void SkipNewlines(NewlineSkipOption skipOption = NewlineSkipOption.None goto again; } - // If the first non-newline, non-whitespace, non-comment, non-backtick, non-semi-colon character - // following the newline is not a pipe, remove the tokens that were added while tokenizing and unget - // the characters that were retrieved. This allows us to "lookahead" to see if there is a pipe and - // only apply line continuance automatically when one is found. - if (skipOption == NewlineSkipOption.BeforePipe && c != '|') - { - var newTokenCount = TokenList != null ? TokenList.Count : 0; - if (newTokenCount > tokenCount) - { - if (tokenCount == 0) - { - TokenList.Clear(); - TokenList = null; - } - else - { - TokenList.RemoveRange(tokenCount - 1, newTokenCount - tokenCount + 1); - } - } - - while (_currentIndex > startIndex) - { - UngetChar(); - } - } - break; } @@ -1002,6 +940,32 @@ private void SkipWhiteSpace() } } + private void ScanNewline(char c) + { + if (TokenList != null) + { + _tokenStart = _currentIndex - 1; + SkipNewline(c); + NewToken(TokenKind.NewLine); + } + } + + private void ScanSemicolon() + { + if (TokenList != null) + { + _tokenStart = _currentIndex - 1; + NewToken(TokenKind.Semi); + } + } + + private void ScanLineContinuation(char c) + { + _tokenStart = _currentIndex - 2; + SkipNewline(c); + NewToken(TokenKind.LineContinuation); + } + internal int GetRestorePoint() { _tokenStart = _currentIndex; @@ -1101,7 +1065,7 @@ internal void ReplaceSavedTokens(Token firstOldToken, Token lastOldToken, Token } } - private void ScanNewline(char c) + private void SkipNewline(char c) { if (c == '\r' && PeekChar() == '\n') { @@ -1342,6 +1306,65 @@ private bool OnlyWhitespaceOrCommentsAfterExtent(InternalScriptExtent extent) return true; } + internal bool IsPipeContinuance(IScriptExtent extent) + { + var scriptExtent = (InternalScriptExtent)extent; + return scriptExtent.EndOffset < _script.Length && PipeContinuanceAfterExtent(scriptExtent); + } + + private bool PipeContinuanceAfterExtent(InternalScriptExtent extent) + { + // If the first non-newline, non-whitespace, non-comment, non-backtick, non-semi-colon character + // following the newline is a pipe, we have a pipe continuance. + for (int i = extent.EndOffset; i < _script.Length;) + { + if (_script[i] == '#') + { + // SkipLineComment will return the position after the comment end + // which is either at the end of the file, or a cr or lf. + i = SkipLineComment(i + 1); + continue; + } + + if (_script[i] == '<' && (i + 1) < _script.Length && _script[i + 1] == '#') + { + i = SkipBlockComment(i + 2); + continue; + } + + if (_script[i].IsWhitespace()) + { + i++; + continue; + } + + if (_script[i] == '\n' || _script[i] == '\r') + { + i = SkipNewline(i); + continue; + } + + if (_script[i] == '`' && (i + 1) < _script.Length) + { + if (_script[i + 1] == '\n' || _script[i + 1] == '\r') + { + i = SkipNewline(i + 1); + continue; + } + + if (char.IsWhiteSpace(_script[i + 1])) + { + i += 2; + continue; + } + } + + return _script[i] == '|'; + } + + return false; + } + private int SkipLineComment(int i) { for (; i < _script.Length; ++i) @@ -1372,6 +1395,25 @@ private int SkipBlockComment(int i) return i; } + private int SkipNewline(int i) + { + if (i < _script.Length) + { + char c = _script[i]; + + if (c == '\r' && (i + 1) < _script.Length && _script[i + 1] == '\n') + { + return i + 2; + } + else if (c == '\n') + { + return i + 1; + } + } + + return i; + } + private char Backtick(char c, out char surrogateCharacter) { surrogateCharacter = s_invalidChar; @@ -1706,7 +1748,7 @@ private void ScanBlockComment() if (c == '\r' || c == '\n') { - ScanNewline(c); + SkipNewline(c); } else if (c == '\0' && AtEof()) { @@ -2482,7 +2524,7 @@ private bool ScanAfterHereStringHeader(string header) if (c == '\r' || c == '\n') { - ScanNewline(c); + SkipNewline(c); } else { @@ -4409,14 +4451,14 @@ internal Token NextToken() case '\r': case '\n': - ScanNewline(c); + SkipNewline(c); return NewToken(TokenKind.NewLine); case '`': c1 = GetChar(); if (c1 == '\n' || c1 == '\r') { - ScanNewline(c1); + SkipNewline(c1); NewToken(TokenKind.LineContinuation); goto again; } From ff03acca214ec0f2ecbac54f1e4e123a4159c442 Mon Sep 17 00:00:00 2001 From: Kirk Munro Date: Wed, 3 Apr 2019 17:09:38 -0300 Subject: [PATCH 10/27] CodeFactor changes --- src/System.Management.Automation/engine/parser/Parser.cs | 3 ++- src/System.Management.Automation/engine/parser/tokenizer.cs | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/System.Management.Automation/engine/parser/Parser.cs b/src/System.Management.Automation/engine/parser/Parser.cs index 039613ff03a..2c0e3c0dee1 100644 --- a/src/System.Management.Automation/engine/parser/Parser.cs +++ b/src/System.Management.Automation/engine/parser/Parser.cs @@ -4254,7 +4254,8 @@ private StatementAst ClassDefinitionRule(List customAttributes if (rCurly.Kind != TokenKind.RCurly) { UngetToken(rCurly); - ReportIncompleteInput(After(lCurly), + ReportIncompleteInput( + After(lCurly), rCurly.Extent, nameof(ParserStrings.MissingEndCurlyBrace), ParserStrings.MissingEndCurlyBrace); diff --git a/src/System.Management.Automation/engine/parser/tokenizer.cs b/src/System.Management.Automation/engine/parser/tokenizer.cs index f992692a8ce..d0083f5fd2e 100644 --- a/src/System.Management.Automation/engine/parser/tokenizer.cs +++ b/src/System.Management.Automation/engine/parser/tokenizer.cs @@ -852,7 +852,6 @@ internal void SkipNewlines(bool skipSemis) { // We normally don't create any tokens here, but the V2 tokenizer api returns newline tokens, // so we create them when asked to create them. - again: char c = GetChar(); switch (c) From 642d6a39bea0ea4eaa32cbddf5ccab721dacc813 Mon Sep 17 00:00:00 2001 From: Kirk Munro Date: Wed, 3 Apr 2019 17:29:14 -0300 Subject: [PATCH 11/27] removing unnecessary/unintentional change --- src/System.Management.Automation/engine/parser/Parser.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/System.Management.Automation/engine/parser/Parser.cs b/src/System.Management.Automation/engine/parser/Parser.cs index 2c0e3c0dee1..d8c73ac9689 100644 --- a/src/System.Management.Automation/engine/parser/Parser.cs +++ b/src/System.Management.Automation/engine/parser/Parser.cs @@ -5820,9 +5820,9 @@ private PipelineBaseAst PipelineRule() switch (nextToken.Kind) { case TokenKind.Semi: + case TokenKind.NewLine: case TokenKind.RParen: case TokenKind.RCurly: - case TokenKind.NewLine: case TokenKind.EndOfInput: scanning = false; continue; From afe285377c309ca242d8ecb4d285f5e0e831861b Mon Sep 17 00:00:00 2001 From: Ilya Date: Fri, 5 Apr 2019 09:34:44 -0300 Subject: [PATCH 12/27] Add argument qualifier to make code more readable Co-Authored-By: KirkMunro --- src/System.Management.Automation/engine/parser/Parser.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/System.Management.Automation/engine/parser/Parser.cs b/src/System.Management.Automation/engine/parser/Parser.cs index d8c73ac9689..e11131573e0 100644 --- a/src/System.Management.Automation/engine/parser/Parser.cs +++ b/src/System.Management.Automation/engine/parser/Parser.cs @@ -357,7 +357,7 @@ private void SkipNewlinesAndSemicolons() if (_ungotToken == null || _ungotToken.Kind == TokenKind.NewLine || _ungotToken.Kind == TokenKind.Semi) { _ungotToken = null; - _tokenizer.SkipNewlines(true); + _tokenizer.SkipNewlines(skipSemis: true); } } From 92ced39cf7c01b6e5af3837986b0d84e56e71ea2 Mon Sep 17 00:00:00 2001 From: Ilya Date: Fri, 5 Apr 2019 09:34:58 -0300 Subject: [PATCH 13/27] Add argument qualifier to make code more readable Co-Authored-By: KirkMunro --- src/System.Management.Automation/engine/parser/Parser.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/System.Management.Automation/engine/parser/Parser.cs b/src/System.Management.Automation/engine/parser/Parser.cs index e11131573e0..d949a826e16 100644 --- a/src/System.Management.Automation/engine/parser/Parser.cs +++ b/src/System.Management.Automation/engine/parser/Parser.cs @@ -348,7 +348,7 @@ private void SkipNewlines() if (_ungotToken == null || _ungotToken.Kind == TokenKind.NewLine) { _ungotToken = null; - _tokenizer.SkipNewlines(false); + _tokenizer.SkipNewlines(skipSemis: false); } } From cc06b3bfa07754d3f1329f63ee07d09d2a9452c5 Mon Sep 17 00:00:00 2001 From: Kirk Munro Date: Fri, 5 Apr 2019 14:56:04 -0300 Subject: [PATCH 14/27] refactor based on PR feedback --- .../engine/parser/tokenizer.cs | 66 +++++++++---------- 1 file changed, 30 insertions(+), 36 deletions(-) diff --git a/src/System.Management.Automation/engine/parser/tokenizer.cs b/src/System.Management.Automation/engine/parser/tokenizer.cs index d0083f5fd2e..69e20b3d8dd 100644 --- a/src/System.Management.Automation/engine/parser/tokenizer.cs +++ b/src/System.Management.Automation/engine/parser/tokenizer.cs @@ -1315,53 +1315,66 @@ private bool PipeContinuanceAfterExtent(InternalScriptExtent extent) { // If the first non-newline, non-whitespace, non-comment, non-backtick, non-semi-colon character // following the newline is a pipe, we have a pipe continuance. - for (int i = extent.EndOffset; i < _script.Length;) + for (int i = extent.EndOffset; i < _script.Length - 1;) { - if (_script[i] == '#') + char c = _script[i]; + + if (c.IsWhitespace()) { - // SkipLineComment will return the position after the comment end - // which is either at the end of the file, or a cr or lf. - i = SkipLineComment(i + 1); + i++; continue; } - if (_script[i] == '<' && (i + 1) < _script.Length && _script[i + 1] == '#') + if (c == '\n') { - i = SkipBlockComment(i + 2); + i++; + continue; + } + else if (c == '\r' && _script[i + 1] == '\n') + { + i += 2; continue; } - if (_script[i].IsWhitespace()) + if (c == '#') { - i++; + // SkipLineComment will return the position after the comment end + // which is either at the end of the file, or a cr or lf. + i = SkipLineComment(i + 1); continue; } - if (_script[i] == '\n' || _script[i] == '\r') + if (c == '<' && _script[i + 1] == '#') { - i = SkipNewline(i); + i = SkipBlockComment(i + 2); continue; } - if (_script[i] == '`' && (i + 1) < _script.Length) + if (c == '`') { - if (_script[i + 1] == '\n' || _script[i + 1] == '\r') + char c2 = _script[i + 1]; + if (c2 == '\n') + { + i += 2; + continue; + } + else if (c2 == '\r' && i + 2 < _script.Length && _script[i + 2] == '\n') { - i = SkipNewline(i + 1); + i += 3; continue; } - if (char.IsWhiteSpace(_script[i + 1])) + if (char.IsWhiteSpace(c2)) { i += 2; continue; } } - return _script[i] == '|'; + return c == '|'; } - return false; + return _script[_script.Length - 1] == '|'; } private int SkipLineComment(int i) @@ -1394,25 +1407,6 @@ private int SkipBlockComment(int i) return i; } - private int SkipNewline(int i) - { - if (i < _script.Length) - { - char c = _script[i]; - - if (c == '\r' && (i + 1) < _script.Length && _script[i + 1] == '\n') - { - return i + 2; - } - else if (c == '\n') - { - return i + 1; - } - } - - return i; - } - private char Backtick(char c, out char surrogateCharacter) { surrogateCharacter = s_invalidChar; From 9268e2f5c26746f17767b7cfd08aa78fe93ce944 Mon Sep 17 00:00:00 2001 From: Kirk Munro Date: Mon, 8 Apr 2019 11:37:44 -0300 Subject: [PATCH 15/27] automatic continuance requires contiguous code --- .../engine/parser/tokenizer.cs | 16 ++++++++ .../Language/Parser/LineContinuance.Tests.ps1 | 37 +++++++++++++------ 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/src/System.Management.Automation/engine/parser/tokenizer.cs b/src/System.Management.Automation/engine/parser/tokenizer.cs index 69e20b3d8dd..a6602fdeca9 100644 --- a/src/System.Management.Automation/engine/parser/tokenizer.cs +++ b/src/System.Management.Automation/engine/parser/tokenizer.cs @@ -1315,6 +1315,8 @@ private bool PipeContinuanceAfterExtent(InternalScriptExtent extent) { // If the first non-newline, non-whitespace, non-comment, non-backtick, non-semi-colon character // following the newline is a pipe, we have a pipe continuance. + bool lastNonWhitespaceIsNewline = true; + for (int i = extent.EndOffset; i < _script.Length - 1;) { char c = _script[i]; @@ -1327,15 +1329,29 @@ private bool PipeContinuanceAfterExtent(InternalScriptExtent extent) if (c == '\n') { + if (lastNonWhitespaceIsNewline) + { + // blank or whitespace-only lines are not allowed in automatic line continuance + return false; + } + lastNonWhitespaceIsNewline = true; i++; continue; } else if (c == '\r' && _script[i + 1] == '\n') { + if (lastNonWhitespaceIsNewline) + { + // blank or whitespace-only lines are not allowed in automatic line continuance + return false; + } + lastNonWhitespaceIsNewline = true; i += 2; continue; } + lastNonWhitespaceIsNewline = false; + if (c == '#') { // SkipLineComment will return the position after the comment end diff --git a/test/powershell/Language/Parser/LineContinuance.Tests.ps1 b/test/powershell/Language/Parser/LineContinuance.Tests.ps1 index bbebd2b73c7..605982cf041 100644 --- a/test/powershell/Language/Parser/LineContinuance.Tests.ps1 +++ b/test/powershell/Language/Parser/LineContinuance.Tests.ps1 @@ -7,6 +7,8 @@ Describe 'Line Continuance' -Tags 'CI' { param ([string]$command) [powershell]::Create().AddScript($command).Invoke() } + + $whitespace = "`t `f`v$([char]0x00a0)$([char]0x0085)" } Context 'Lines ending with a backtick that parse and execute without error' { @@ -22,7 +24,7 @@ Describe 'Line Continuance' -Tags 'CI' { $script = @' # The first line of this command ends with trailing whitespace 'Hello' + ` -'@ + ' ' + @' +'@ + $whitespace + @' ' world' '@ @@ -80,7 +82,7 @@ Describe 'Line Continuance' -Tags 'CI' { $script = @' # The next line ends with trailing whitespace 'Hello' | -'@ + ' ' + @' +'@ + $whitespace + @' ForEach-Object {"$_ world"} '@ @@ -167,16 +169,6 @@ Describe 'Line Continuance' -Tags 'CI' { ExecuteCommand $script | Should -Be @(30, 24, 18, 12, 6) } - It 'Line continuance using a pipe at the start of a subsequent line after multiple blank lines' { - $script = @' -'Hello' - - - | ForEach-Object {"$_ world"} -'@ - ExecuteCommand $script | Should -Be 'Hello world' - } - It 'Line continuance using a pipe on a line by itself' { $script = @' 'Hello' @@ -248,5 +240,26 @@ Describe 'Line Continuance' -Tags 'CI' { $err = { ExecuteCommand $script } | Should -Throw -ErrorId 'ParseException' -PassThru $err.Exception.InnerException.ErrorRecord.FullyQualifiedErrorId | Should -BeExactly 'EmptyPipeElement' } + + It 'Lines starting with a single pipe that have a blank line before it' { + $script = @' +'Hello' + + | ForEach-Object {"$_ world"} + +'@ + $err = { ExecuteCommand $script } | Should -Throw -ErrorId 'ParseException' -PassThru + $err.Exception.InnerException.ErrorRecord.FullyQualifiedErrorId | Should -BeExactly 'EmptyPipeElement' + } + + It 'Lines starting with a single pipe that have a line with whitespace before it' { + $script = @" +'Hello' +$whitespace + | ForEach-Object {"`$_ world"} +"@ + $err = { ExecuteCommand $script } | Should -Throw -ErrorId 'ParseException' -PassThru + $err.Exception.InnerException.ErrorRecord.FullyQualifiedErrorId | Should -BeExactly 'EmptyPipeElement' + } } } From b31ec46fcbdfcfa53b3825fcff3f22a30afaf663 Mon Sep 17 00:00:00 2001 From: Kirk Munro Date: Fri, 26 Apr 2019 17:37:15 -0300 Subject: [PATCH 16/27] CodeFactor changes --- src/System.Management.Automation/engine/parser/tokenizer.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/System.Management.Automation/engine/parser/tokenizer.cs b/src/System.Management.Automation/engine/parser/tokenizer.cs index e1a3c973ec7..d85a92ed59b 100644 --- a/src/System.Management.Automation/engine/parser/tokenizer.cs +++ b/src/System.Management.Automation/engine/parser/tokenizer.cs @@ -1360,6 +1360,7 @@ private bool PipeContinuanceAfterExtent(InternalScriptExtent extent) // blank or whitespace-only lines are not allowed in automatic line continuance return false; } + lastNonWhitespaceIsNewline = true; i++; continue; @@ -1371,6 +1372,7 @@ private bool PipeContinuanceAfterExtent(InternalScriptExtent extent) // blank or whitespace-only lines are not allowed in automatic line continuance return false; } + lastNonWhitespaceIsNewline = true; i += 2; continue; From ad15226aa09e2f3c75fb04346866479e64a9c5b9 Mon Sep 17 00:00:00 2001 From: Kirk Munro Date: Tue, 7 May 2019 17:16:58 -0300 Subject: [PATCH 17/27] changes for PR feedback; added some Pester tests --- .../engine/parser/tokenizer.cs | 66 ++++++++++++------- .../Language/Parser/LineContinuance.Tests.ps1 | 15 +++++ 2 files changed, 56 insertions(+), 25 deletions(-) diff --git a/src/System.Management.Automation/engine/parser/tokenizer.cs b/src/System.Management.Automation/engine/parser/tokenizer.cs index bf4731b36e1..7f0d7d6c77d 100644 --- a/src/System.Management.Automation/engine/parser/tokenizer.cs +++ b/src/System.Management.Automation/engine/parser/tokenizer.cs @@ -7,7 +7,6 @@ using System.Diagnostics; using System.Globalization; using System.Linq; -using System.Linq.Expressions; using System.Numerics; using System.Runtime.CompilerServices; using System.Text; @@ -877,8 +876,9 @@ internal static bool IsKeyword(string str) internal void SkipNewlines(bool skipSemis) { - // We normally don't create any tokens here, but the V2 tokenizer api returns newline tokens, - // so we create them when asked to create them. + // We normally don't create any tokens in a Skip method, but the + // V2 tokenizer api returns newline, semi-colon, and line + // continuation tokens so we create them as they are encountered. again: char c = GetChar(); switch (c) @@ -968,19 +968,21 @@ private void SkipWhiteSpace() private void ScanNewline(char c) { + _tokenStart = _currentIndex - 1; + NormalizeCRLF(c); + // Memory optimization: only create the token if it will be stored if (TokenList != null) { - _tokenStart = _currentIndex - 1; - SkipNewline(c); NewToken(TokenKind.NewLine); } } private void ScanSemicolon() { + _tokenStart = _currentIndex - 1; + // Memory optimization: only create the token if it will be stored if (TokenList != null) { - _tokenStart = _currentIndex - 1; NewToken(TokenKind.Semi); } } @@ -988,8 +990,12 @@ private void ScanSemicolon() private void ScanLineContinuation(char c) { _tokenStart = _currentIndex - 2; - SkipNewline(c); - NewToken(TokenKind.LineContinuation); + NormalizeCRLF(c); + // Memory optimization: only create the token if it will be stored + if (TokenList != null) + { + NewToken(TokenKind.LineContinuation); + } } internal int GetRestorePoint() @@ -1091,8 +1097,9 @@ internal void ReplaceSavedTokens(Token firstOldToken, Token lastOldToken, Token } } - private void SkipNewline(char c) + private void NormalizeCRLF(char c) { + // CRs in Windows line endings are ignored if (c == '\r' && PeekChar() == '\n') { SkipChar(); @@ -1334,17 +1341,21 @@ private bool OnlyWhitespaceOrCommentsAfterExtent(InternalScriptExtent extent) internal bool IsPipeContinuance(IScriptExtent extent) { - var scriptExtent = (InternalScriptExtent)extent; - return scriptExtent.EndOffset < _script.Length && PipeContinuanceAfterExtent(scriptExtent); + return extent.EndOffset < _script.Length && PipeContinuanceAfterExtent(extent); } - private bool PipeContinuanceAfterExtent(InternalScriptExtent extent) + private bool PipeContinuanceAfterExtent(IScriptExtent extent) { // If the first non-newline, non-whitespace, non-comment, non-backtick, non-semi-colon character // following the newline is a pipe, we have a pipe continuance. bool lastNonWhitespaceIsNewline = true; + int i = extent.EndOffset; - for (int i = extent.EndOffset; i < _script.Length - 1;) + // Since some token pattern matching looks for multiple characters (e.g. newline, block comment, + // or line continuance), we stop searching at _script.Length - 1 and perform one additional check + // after the while loop. This avoids having to compare i + 1 against the script length in multiple + // locations inside the loop. + while (i < _script.Length - 1) { char c = _script[i]; @@ -1366,7 +1377,7 @@ private bool PipeContinuanceAfterExtent(InternalScriptExtent extent) i++; continue; } - else if (c == '\r' && _script[i + 1] == '\n') + else if (c == '\r') { if (lastNonWhitespaceIsNewline) { @@ -1375,7 +1386,7 @@ private bool PipeContinuanceAfterExtent(InternalScriptExtent extent) } lastNonWhitespaceIsNewline = true; - i += 2; + i += _script[i + 1] == '\n' ? 2 : 1; continue; } @@ -1403,9 +1414,9 @@ private bool PipeContinuanceAfterExtent(InternalScriptExtent extent) i += 2; continue; } - else if (c2 == '\r' && i + 2 < _script.Length && _script[i + 2] == '\n') + else if (c2 == '\r') { - i += 3; + i += i + 2 < _script.Length && _script[i + 2] == '\n' ? 3 : 2; continue; } @@ -1784,9 +1795,9 @@ private void ScanBlockComment() break; } - if (c == '\r' || c == '\n') + if (c == '\r') { - SkipNewline(c); + NormalizeCRLF(c); } else if (c == '\0' && AtEof()) { @@ -2560,11 +2571,11 @@ private bool ScanAfterHereStringHeader(string header) c = GetChar(); } while (c.IsWhitespace()); - if (c == '\r' || c == '\n') + if (c == '\r') { - SkipNewline(c); + NormalizeCRLF(c); } - else + else if (c != '\n') { if (c == '\0' && AtEof()) { @@ -4618,16 +4629,21 @@ internal Token NextToken() ScanLineComment(); goto again; - case '\r': case '\n': - SkipNewline(c); return NewToken(TokenKind.NewLine); + case '\r': + NormalizeCRLF(c); + goto case '\n'; + case '`': c1 = GetChar(); + if (c1 == '\r') + { + NormalizeCRLF(c1); + } if (c1 == '\n' || c1 == '\r') { - SkipNewline(c1); NewToken(TokenKind.LineContinuation); goto again; } diff --git a/test/powershell/Language/Parser/LineContinuance.Tests.ps1 b/test/powershell/Language/Parser/LineContinuance.Tests.ps1 index 605982cf041..21fd24e7e80 100644 --- a/test/powershell/Language/Parser/LineContinuance.Tests.ps1 +++ b/test/powershell/Language/Parser/LineContinuance.Tests.ps1 @@ -20,6 +20,11 @@ Describe 'Line Continuance' -Tags 'CI' { ExecuteCommand $script | Should -Be 'Hello world' } + It 'Lines ending with a single backtick followed only by a CR (old-style Mac line ending)' { + $script = "'Hello' + ```r' world'" + ExecuteCommand $script | Should -Be 'Hello world' + } + It 'Lines ending with a single backtick followed by whitespace' { $script = @' # The first line of this command ends with trailing whitespace @@ -78,6 +83,11 @@ Describe 'Line Continuance' -Tags 'CI' { ExecuteCommand $script | Should -Be 'Hello world' } + It 'Lines ending with a pipe followed only by a CR (old-style Mac line ending)' { + $script = "'Hello' |`r ForEach-Object {`"`$_ world`"}" + ExecuteCommand $script | Should -Be 'Hello world' + } + It 'Lines ending with a pipe followed by whitespace' { $script = @' # The next line ends with trailing whitespace @@ -137,6 +147,11 @@ Describe 'Line Continuance' -Tags 'CI' { ExecuteCommand $script | Should -Be 'Hello world' } + It 'Line continuance using a pipe at the start of a subsequent line after a CR (old-style Mac line ending)' { + $script = "'Hello'`r | ForEach-Object {`"`$_ world`"}" + ExecuteCommand $script | Should -Be 'Hello world' + } + It 'Longer line continuance using pipes at the start of subsequent lines' { $script = @' 1..10 From 8f6bb2a77d2c95b27fb7696fd51dd7ffca2f12e8 Mon Sep 17 00:00:00 2001 From: Kirk Munro Date: Mon, 13 May 2019 12:40:06 -0300 Subject: [PATCH 18/27] Code review change Co-Authored-By: Ilya --- src/System.Management.Automation/engine/parser/tokenizer.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/System.Management.Automation/engine/parser/tokenizer.cs b/src/System.Management.Automation/engine/parser/tokenizer.cs index 7f0d7d6c77d..3288decf94d 100644 --- a/src/System.Management.Automation/engine/parser/tokenizer.cs +++ b/src/System.Management.Automation/engine/parser/tokenizer.cs @@ -970,6 +970,7 @@ private void ScanNewline(char c) { _tokenStart = _currentIndex - 1; NormalizeCRLF(c); + // Memory optimization: only create the token if it will be stored if (TokenList != null) { From 6580a943ee9e45636074c6e0317eb1bb143d94a9 Mon Sep 17 00:00:00 2001 From: Kirk Munro Date: Mon, 13 May 2019 12:40:22 -0300 Subject: [PATCH 19/27] Code review change Co-Authored-By: Ilya --- src/System.Management.Automation/engine/parser/tokenizer.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/System.Management.Automation/engine/parser/tokenizer.cs b/src/System.Management.Automation/engine/parser/tokenizer.cs index 3288decf94d..72ec8c79572 100644 --- a/src/System.Management.Automation/engine/parser/tokenizer.cs +++ b/src/System.Management.Automation/engine/parser/tokenizer.cs @@ -981,6 +981,7 @@ private void ScanNewline(char c) private void ScanSemicolon() { _tokenStart = _currentIndex - 1; + // Memory optimization: only create the token if it will be stored if (TokenList != null) { From 71395f8242135287336637dfd7e263b53436cf07 Mon Sep 17 00:00:00 2001 From: Kirk Munro Date: Mon, 13 May 2019 12:40:34 -0300 Subject: [PATCH 20/27] Code review change Co-Authored-By: Ilya --- src/System.Management.Automation/engine/parser/tokenizer.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/System.Management.Automation/engine/parser/tokenizer.cs b/src/System.Management.Automation/engine/parser/tokenizer.cs index 72ec8c79572..ccab3f95fe3 100644 --- a/src/System.Management.Automation/engine/parser/tokenizer.cs +++ b/src/System.Management.Automation/engine/parser/tokenizer.cs @@ -993,6 +993,7 @@ private void ScanLineContinuation(char c) { _tokenStart = _currentIndex - 2; NormalizeCRLF(c); + // Memory optimization: only create the token if it will be stored if (TokenList != null) { From f7f3938f7d44d7fcdb941a17cd15aad5b3be824f Mon Sep 17 00:00:00 2001 From: Kirk Munro Date: Mon, 13 May 2019 12:41:03 -0300 Subject: [PATCH 21/27] Code review change Co-Authored-By: Ilya --- test/powershell/Language/Parser/LineContinuance.Tests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/powershell/Language/Parser/LineContinuance.Tests.ps1 b/test/powershell/Language/Parser/LineContinuance.Tests.ps1 index 21fd24e7e80..a54e4025ba2 100644 --- a/test/powershell/Language/Parser/LineContinuance.Tests.ps1 +++ b/test/powershell/Language/Parser/LineContinuance.Tests.ps1 @@ -144,7 +144,7 @@ Describe 'Line Continuance' -Tags 'CI' { 'Hello' | ForEach-Object {"$_ world"} '@ - ExecuteCommand $script | Should -Be 'Hello world' + ExecuteCommand $script | Should -BeExactly 'Hello world' } It 'Line continuance using a pipe at the start of a subsequent line after a CR (old-style Mac line ending)' { From 01543db57af02e37057c3f2df36ceb3a540be9b0 Mon Sep 17 00:00:00 2001 From: Kirk Munro Date: Mon, 13 May 2019 12:41:13 -0300 Subject: [PATCH 22/27] Code review change Co-Authored-By: Ilya --- test/powershell/Language/Parser/LineContinuance.Tests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/powershell/Language/Parser/LineContinuance.Tests.ps1 b/test/powershell/Language/Parser/LineContinuance.Tests.ps1 index a54e4025ba2..6221562023d 100644 --- a/test/powershell/Language/Parser/LineContinuance.Tests.ps1 +++ b/test/powershell/Language/Parser/LineContinuance.Tests.ps1 @@ -149,7 +149,7 @@ Describe 'Line Continuance' -Tags 'CI' { It 'Line continuance using a pipe at the start of a subsequent line after a CR (old-style Mac line ending)' { $script = "'Hello'`r | ForEach-Object {`"`$_ world`"}" - ExecuteCommand $script | Should -Be 'Hello world' + ExecuteCommand $script | Should -BeExactly 'Hello world' } It 'Longer line continuance using pipes at the start of subsequent lines' { From 090cfb76385063119b3301468a97187b3e41f237 Mon Sep 17 00:00:00 2001 From: Kirk Munro Date: Mon, 13 May 2019 12:41:23 -0300 Subject: [PATCH 23/27] Code review change Co-Authored-By: Ilya --- test/powershell/Language/Parser/LineContinuance.Tests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/powershell/Language/Parser/LineContinuance.Tests.ps1 b/test/powershell/Language/Parser/LineContinuance.Tests.ps1 index 6221562023d..84ff3acd929 100644 --- a/test/powershell/Language/Parser/LineContinuance.Tests.ps1 +++ b/test/powershell/Language/Parser/LineContinuance.Tests.ps1 @@ -168,7 +168,7 @@ Describe 'Line Continuance' -Tags 'CI' { # You can place comments before continued pipelines | ForEach-Object {"$_ world"} '@ - ExecuteCommand $script | Should -Be 'Hello world' + ExecuteCommand $script | Should -BeExactly 'Hello world' } It 'Longer line continuance using pipes at the start of subsequent lines (with comments)' { From 29ce0a77ff1e65300a0d0a670c0229028769eb5f Mon Sep 17 00:00:00 2001 From: Kirk Munro Date: Mon, 13 May 2019 12:41:35 -0300 Subject: [PATCH 24/27] Code review change Co-Authored-By: Ilya --- test/powershell/Language/Parser/LineContinuance.Tests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/powershell/Language/Parser/LineContinuance.Tests.ps1 b/test/powershell/Language/Parser/LineContinuance.Tests.ps1 index 84ff3acd929..d071782929f 100644 --- a/test/powershell/Language/Parser/LineContinuance.Tests.ps1 +++ b/test/powershell/Language/Parser/LineContinuance.Tests.ps1 @@ -190,7 +190,7 @@ Describe 'Line Continuance' -Tags 'CI' { | ForEach-Object {"$_ world"} '@ - ExecuteCommand $script | Should -Be 'Hello world' + ExecuteCommand $script | Should -BeExactly 'Hello world' } From ea1a80c4e8eed3183816674572c9363c7d7585c6 Mon Sep 17 00:00:00 2001 From: Kirk Munro Date: Mon, 13 May 2019 12:41:47 -0300 Subject: [PATCH 25/27] Code review change Co-Authored-By: Ilya --- test/powershell/Language/Parser/LineContinuance.Tests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/powershell/Language/Parser/LineContinuance.Tests.ps1 b/test/powershell/Language/Parser/LineContinuance.Tests.ps1 index d071782929f..dc8083f5a1d 100644 --- a/test/powershell/Language/Parser/LineContinuance.Tests.ps1 +++ b/test/powershell/Language/Parser/LineContinuance.Tests.ps1 @@ -214,7 +214,7 @@ Describe 'Line Continuance' -Tags 'CI' { # You can place comments before continued pipelines ForEach-Object {"$_ world"} '@ - ExecuteCommand $script | Should -Be 'Hello world' + ExecuteCommand $script | Should -BeExactly 'Hello world' } It 'Longer line continuance using pipes on lines by themselves (with comments)' { From 9dc60405dd1f247addc2effc12dab2b3d300198a Mon Sep 17 00:00:00 2001 From: Kirk Munro Date: Mon, 13 May 2019 12:42:03 -0300 Subject: [PATCH 26/27] Code review change Co-Authored-By: Ilya --- test/powershell/Language/Parser/LineContinuance.Tests.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/powershell/Language/Parser/LineContinuance.Tests.ps1 b/test/powershell/Language/Parser/LineContinuance.Tests.ps1 index dc8083f5a1d..c4425cddec1 100644 --- a/test/powershell/Language/Parser/LineContinuance.Tests.ps1 +++ b/test/powershell/Language/Parser/LineContinuance.Tests.ps1 @@ -17,7 +17,7 @@ Describe 'Line Continuance' -Tags 'CI' { 'Hello' + ` ' world' '@ - ExecuteCommand $script | Should -Be 'Hello world' + ExecuteCommand $script | Should -BeExactly 'Hello world' } It 'Lines ending with a single backtick followed only by a CR (old-style Mac line ending)' { From b4e1a8b767242fc4fb4b58cb1867785f48b838fa Mon Sep 17 00:00:00 2001 From: Kirk Munro Date: Mon, 13 May 2019 18:37:50 -0300 Subject: [PATCH 27/27] changes for code review --- .../engine/parser/tokenizer.cs | 33 ++++--------------- .../Language/Parser/LineContinuance.Tests.ps1 | 12 +++++++ 2 files changed, 18 insertions(+), 27 deletions(-) diff --git a/src/System.Management.Automation/engine/parser/tokenizer.cs b/src/System.Management.Automation/engine/parser/tokenizer.cs index ccab3f95fe3..9fcc1bddecb 100644 --- a/src/System.Management.Automation/engine/parser/tokenizer.cs +++ b/src/System.Management.Automation/engine/parser/tokenizer.cs @@ -1349,15 +1349,15 @@ internal bool IsPipeContinuance(IScriptExtent extent) private bool PipeContinuanceAfterExtent(IScriptExtent extent) { - // If the first non-newline, non-whitespace, non-comment, non-backtick, non-semi-colon character - // following the newline is a pipe, we have a pipe continuance. + // If the first non-comment (regular or block) character following a newline is a pipe, we have + // pipe continuance. bool lastNonWhitespaceIsNewline = true; int i = extent.EndOffset; - // Since some token pattern matching looks for multiple characters (e.g. newline, block comment, - // or line continuance), we stop searching at _script.Length - 1 and perform one additional check - // after the while loop. This avoids having to compare i + 1 against the script length in multiple - // locations inside the loop. + // Since some token pattern matching looks for multiple characters (e.g. newline or block comment) + // we stop searching at _script.Length - 1 and perform one additional check after the while loop. + // This avoids having to compare i + 1 against the script length in multiple locations inside the + // loop. while (i < _script.Length - 1) { char c = _script[i]; @@ -1409,27 +1409,6 @@ private bool PipeContinuanceAfterExtent(IScriptExtent extent) continue; } - if (c == '`') - { - char c2 = _script[i + 1]; - if (c2 == '\n') - { - i += 2; - continue; - } - else if (c2 == '\r') - { - i += i + 2 < _script.Length && _script[i + 2] == '\n' ? 3 : 2; - continue; - } - - if (char.IsWhiteSpace(c2)) - { - i += 2; - continue; - } - } - return c == '|'; } diff --git a/test/powershell/Language/Parser/LineContinuance.Tests.ps1 b/test/powershell/Language/Parser/LineContinuance.Tests.ps1 index c4425cddec1..2bb5395119e 100644 --- a/test/powershell/Language/Parser/LineContinuance.Tests.ps1 +++ b/test/powershell/Language/Parser/LineContinuance.Tests.ps1 @@ -276,5 +276,17 @@ $whitespace $err = { ExecuteCommand $script } | Should -Throw -ErrorId 'ParseException' -PassThru $err.Exception.InnerException.ErrorRecord.FullyQualifiedErrorId | Should -BeExactly 'EmptyPipeElement' } + + It 'Lines starting with a single pipe that have a line with nothing but a backtick before it' { + $script = @' +'Hello' + ` + | ForEach-Object {"$_ world"} + +'@ + $err = { ExecuteCommand $script } | Should -Throw -ErrorId 'ParseException' -PassThru + $err.Exception.InnerException.ErrorRecord.FullyQualifiedErrorId | Should -BeExactly 'EmptyPipeElement' + } + } }