From bcc2719ccd81d73eb7ce4064cd4543e418bf0054 Mon Sep 17 00:00:00 2001 From: James Truher Date: Mon, 2 Oct 2017 14:57:23 -0700 Subject: [PATCH 1/8] Unify cmdlets with parameter 'Encoding' to be of type System.Text.Encoding Created support classes so tab completion would continue to wor Create an ArgumentTransformationAttribute for the parameter to accept the currently used strings Register encoding providers on all platforms. This seems safe enough, and there are defintely more encodings available on CoreFX after this is done, which will be useful for some of the web cmdlet scenarios Validate that files are created with UTF-8 encoding without BOM Update tests to validate Encoding parameter to new type and create new tests for parameter type validation. --- .../commands/utility/CSVCommands.cs | 48 ++++++- .../FormatAndOutput/format-hex/Format-Hex.cs | 29 ++-- .../FormatAndOutput/out-file/Out-File.cs | 19 +-- .../utility/ImplicitRemotingCommands.cs | 18 ++- .../commands/utility/MatchString.cs | 45 +++--- .../commands/utility/Send-MailMessage.cs | 2 +- .../commands/utility/XmlCommands.cs | 20 ++- .../engine/Utils.cs | 82 +++-------- .../engine/hostifaces/MshHostUserInterface.cs | 10 +- .../namespaces/FileSystemProvider.cs | 124 +++++----------- .../utils/ClrFacade.cs | 18 +-- .../utils/PathUtils.cs | 134 +++++++++++++++++- .../Parser/RedirectionOperator.Tests.ps1 | 11 +- .../TestGetCommand.Tests.ps1 | 8 +- .../Get-Content.Tests.ps1 | 5 +- .../engine/Basic/Encoding.Tests.ps1 | 74 ++++++++++ 16 files changed, 398 insertions(+), 249 deletions(-) create mode 100644 test/powershell/engine/Basic/Encoding.Tests.ps1 diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/CSVCommands.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/CSVCommands.cs index ca7752da8bc..b5d456e4e6c 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/CSVCommands.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/CSVCommands.cs @@ -212,8 +212,27 @@ public SwitchParameter NoClobber /// Encoding optional flag /// [Parameter()] - [ValidateSetAttribute(new string[] { "Unicode", "UTF7", "UTF8", "ASCII", "UTF32", "BigEndianUnicode", "Default", "OEM" })] - public string Encoding { get; set; } + [ArgumentToEncodingTransformationAttribute()] + [ArgumentCompleter(typeof(EncodingArgumentCompleter))] + public Encoding Encoding + { + get + { + return _encoding; + } + set + { + if ( value == EncodingConversion.byteEncoding ) + { + _encoding = EncodingConversion.byteEncoding.ActualEncoding; + } + else + { + _encoding = value; + } + } + } + private Encoding _encoding = ClrFacade.GetDefaultEncoding(); /// /// Property that sets append parameter. @@ -373,7 +392,7 @@ private void CreateFileStream() PathUtils.MasterStreamOpen( this, this.Path, - Encoding ?? "ASCII", + Encoding, false, // defaultEncoding Append, Force, @@ -577,8 +596,27 @@ public SwitchParameter UseCulture /// Encoding optional flag /// [Parameter()] - [ValidateSetAttribute(new[] { "Unicode", "UTF7", "UTF8", "ASCII", "UTF32", "BigEndianUnicode", "Default", "OEM" })] - public string Encoding { get; set; } + [ArgumentToEncodingTransformationAttribute()] + [ArgumentCompleter(typeof(EncodingArgumentCompleter))] + public Encoding Encoding + { + get + { + return _encoding; + } + set + { + if ( value == EncodingConversion.byteEncoding ) + { + _encoding = EncodingConversion.byteEncoding.ActualEncoding; + } + else + { + _encoding = value; + } + } + } + private Encoding _encoding = ClrFacade.GetDefaultEncoding(); /// /// Avoid writing out duplicate warning messages when there are diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/format-hex/Format-Hex.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/format-hex/Format-Hex.cs index b1c65d3f2b1..fe0a3174fa6 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/format-hex/Format-Hex.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/format-hex/Format-Hex.cs @@ -46,14 +46,24 @@ public sealed class FormatHex : PSCmdlet /// Type of character encoding for InputObject /// [Parameter(ParameterSetName = "ByInputObject")] - [ValidateSetAttribute(new string[] { - EncodingConversion.Unicode, - EncodingConversion.BigEndianUnicode, - EncodingConversion.Utf8, - EncodingConversion.Utf7, - EncodingConversion.Utf32, - EncodingConversion.Ascii})] - public string Encoding { get; set; } = "Ascii"; + [ArgumentToEncodingTransformationAttribute()] + [ArgumentCompleter(typeof(EncodingArgumentCompleter))] + public Encoding Encoding + { + get { return _encoding; } + set + { + if ( value == EncodingConversion.byteEncoding ) + { + _encoding = EncodingConversion.byteEncoding.ActualEncoding; + } + else + { + _encoding = value; + } + } + } + private Encoding _encoding = ClrFacade.GetDefaultEncoding(); /// /// This parameter is no-op @@ -239,8 +249,7 @@ private void ProcessObjectContent(PSObject inputObject) else if (obj is string) { string inputString = obj.ToString(); - Encoding resolvedEncoding = EncodingConversion.Convert(this, Encoding); - inputBytes = resolvedEncoding.GetBytes(inputString); + inputBytes = Encoding.GetBytes(inputString); } else if (obj is byte) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/out-file/Out-File.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/out-file/Out-File.cs index b8fd1cc09b6..79aab995105 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/out-file/Out-File.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/out-file/Out-File.cs @@ -3,6 +3,7 @@ --********************************************************************/ using System; +using System.Text; using System.Management.Automation; using System.Management.Automation.Internal; using System.Management.Automation.Host; @@ -72,25 +73,15 @@ public string LiteralPath /// /// [Parameter(Position = 1)] - [ValidateNotNullOrEmpty] - [ValidateSetAttribute(new string[] { - EncodingConversion.Unknown, - EncodingConversion.String, - EncodingConversion.Unicode, - EncodingConversion.BigEndianUnicode, - EncodingConversion.Utf8, - EncodingConversion.Utf7, - EncodingConversion.Utf32, - EncodingConversion.Ascii, - EncodingConversion.Default, - EncodingConversion.OEM })] - public string Encoding + [ArgumentToEncodingTransformationAttribute()] + [ArgumentCompleter(typeof(EncodingArgumentCompleter))] + public Encoding Encoding { get { return _encoding; } set { _encoding = value; } } - private string _encoding; + private Encoding _encoding = ClrFacade.GetDefaultEncoding(); /// /// Property that sets append parameter. diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/ImplicitRemotingCommands.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/ImplicitRemotingCommands.cs index e694403c6b8..fd7d62c0cc0 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/ImplicitRemotingCommands.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/ImplicitRemotingCommands.cs @@ -77,19 +77,27 @@ public SwitchParameter Force /// Encoding optional flag /// [Parameter] - [ValidateSetAttribute(new string[] { "Unicode", "UTF7", "UTF8", "ASCII", "UTF32", "BigEndianUnicode", "Default", "OEM" })] - public string Encoding + [ArgumentToEncodingTransformationAttribute()] + [ArgumentCompleter(typeof(EncodingArgumentCompleter))] + public Encoding Encoding { get { - return _encoding.GetType().Name; + return _encoding; } set { - _encoding = EncodingConversion.Convert(this, value); + if ( value == EncodingConversion.byteEncoding ) + { + _encoding = EncodingConversion.byteEncoding.ActualEncoding; + } + else + { + _encoding = value; + } } } - private Encoding _encoding = System.Text.Encoding.UTF8; + private Encoding _encoding = ClrFacade.GetDefaultEncoding(); #endregion Parameters diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/MatchString.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/MatchString.cs index deaa37323db..813e1f8c0c8 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/MatchString.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/MatchString.cs @@ -3,6 +3,7 @@ --********************************************************************/ using System; +using System.Text; using System.Text.RegularExpressions; using System.IO; using System.Collections; @@ -1201,19 +1202,27 @@ public SwitchParameter AllMatches /// The text encoding to process each file as. /// [Parameter] - [ValidateNotNullOrEmpty] - [ValidateSetAttribute(new string[] { - EncodingConversion.Unicode, - EncodingConversion.Utf7, - EncodingConversion.Utf8, - EncodingConversion.Utf32, - EncodingConversion.Ascii, - EncodingConversion.BigEndianUnicode, - EncodingConversion.Default, - EncodingConversion.OEM })] - public string Encoding { get; set; } - - private System.Text.Encoding _textEncoding; + [ArgumentToEncodingTransformationAttribute()] + [ArgumentCompleter(typeof(EncodingArgumentCompleter))] + public Encoding Encoding + { + get + { + return _textEncoding; + } + set + { + if ( value == EncodingConversion.byteEncoding ) + { + _textEncoding = EncodingConversion.byteEncoding.ActualEncoding; + } + else + { + _textEncoding = value; + } + } + } + private System.Text.Encoding _textEncoding = ClrFacade.GetDefaultEncoding(); /// /// The number of context lines to collect. If set to a @@ -1282,16 +1291,6 @@ public SwitchParameter AllMatches /// protected override void BeginProcessing() { - // Process encoding switch. - if (Encoding != null) - { - _textEncoding = EncodingConversion.Convert(this, Encoding); - } - else - { - _textEncoding = new System.Text.UTF8Encoding(); - } - if (!_simpleMatch) { RegexOptions regexOptions = (_caseSensitive) ? RegexOptions.None : RegexOptions.IgnoreCase; diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Send-MailMessage.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Send-MailMessage.cs index b2667207242..a63ad2700e0 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Send-MailMessage.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Send-MailMessage.cs @@ -523,4 +523,4 @@ public override object Transform(EngineIntrinsics engineIntrinsics, object input } #endregion -} \ No newline at end of file +} diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/XmlCommands.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/XmlCommands.cs index 50edf754d29..3aa4bef7608 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/XmlCommands.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/XmlCommands.cs @@ -108,8 +108,24 @@ public SwitchParameter NoClobber /// /// [Parameter] - [ValidateSetAttribute(new string[] { "Unicode", "UTF7", "UTF8", "ASCII", "UTF32", "BigEndianUnicode", "Default", "OEM" })] - public string Encoding { get; set; } = "Unicode"; + [ArgumentToEncodingTransformationAttribute()] + [ArgumentCompleter(typeof(EncodingArgumentCompleter))] + public Encoding Encoding + { + get { return _encoding; } + set + { + if ( value == EncodingConversion.byteEncoding ) + { + _encoding = EncodingConversion.byteEncoding.ActualEncoding; + } + else + { + _encoding = value; + } + } + } + private Encoding _encoding = ClrFacade.GetDefaultEncoding(); #endregion Command Line Parameters diff --git a/src/System.Management.Automation/engine/Utils.cs b/src/System.Management.Automation/engine/Utils.cs index dc50286acf9..61a8baab95d 100644 --- a/src/System.Management.Automation/engine/Utils.cs +++ b/src/System.Management.Automation/engine/Utils.cs @@ -1147,11 +1147,12 @@ internal static bool Succeeded(int hresult) return hresult >= 0; } - internal static FileSystemCmdletProviderEncoding GetEncoding(string path) + // Attempt to determine the existing encoding + internal static Encoding GetEncoding(string path) { if (!File.Exists(path)) { - return FileSystemCmdletProviderEncoding.Default; + return ClrFacade.GetDefaultEncoding(); } byte[] initialBytes = new byte[100]; @@ -1169,12 +1170,12 @@ internal static FileSystemCmdletProviderEncoding GetEncoding(string path) } catch (IOException) { - return FileSystemCmdletProviderEncoding.Default; + return ClrFacade.GetDefaultEncoding(); } // Test for four-byte preambles string preamble = null; - FileSystemCmdletProviderEncoding foundEncoding = FileSystemCmdletProviderEncoding.Default; + Encoding foundEncoding = ClrFacade.GetDefaultEncoding(); if (bytesRead > 3) { @@ -1210,77 +1211,26 @@ internal static FileSystemCmdletProviderEncoding GetEncoding(string path) string initialBytesAsAscii = System.Text.Encoding.ASCII.GetString(initialBytes, 0, bytesRead); if (initialBytesAsAscii.IndexOfAny(nonPrintableCharacters) >= 0) { - return FileSystemCmdletProviderEncoding.Byte; + return Encoding.Unicode; } - return FileSystemCmdletProviderEncoding.Ascii; + return Encoding.ASCII; } - internal static Encoding GetEncodingFromEnum(FileSystemCmdletProviderEncoding encoding) - { - // Default to unicode encoding - Encoding result = Encoding.Unicode; - - switch (encoding) - { - case FileSystemCmdletProviderEncoding.String: - result = Encoding.Unicode; - break; - - case FileSystemCmdletProviderEncoding.Unicode: - result = Encoding.Unicode; - break; - - case FileSystemCmdletProviderEncoding.BigEndianUnicode: - result = Encoding.BigEndianUnicode; - break; - - case FileSystemCmdletProviderEncoding.UTF8: - result = Encoding.UTF8; - break; - - case FileSystemCmdletProviderEncoding.UTF7: - result = Encoding.UTF7; - break; - - case FileSystemCmdletProviderEncoding.UTF32: - result = Encoding.UTF32; - break; - - case FileSystemCmdletProviderEncoding.BigEndianUTF32: - result = Encoding.BigEndianUnicode; - break; - - case FileSystemCmdletProviderEncoding.Ascii: - result = Encoding.ASCII; - break; - - case FileSystemCmdletProviderEncoding.Default: - result = ClrFacade.GetDefaultEncoding(); - break; - - case FileSystemCmdletProviderEncoding.Oem: - result = ClrFacade.GetOEMEncoding(); - break; - - default: - break; - } - - return result; - } // GetEncodingFromEnum + // BigEndianUTF32 encoding is possible, but requires creation + internal static Encoding BigEndianUTF32Encoding = new UTF32Encoding(true,true); // [System.Text.Encoding]::GetEncodings() | Where-Object { $_.GetEncoding().GetPreamble() } | // Add-Member ScriptProperty Preamble { $this.GetEncoding().GetPreamble() -join "-" } -PassThru | // Format-Table -Auto - internal static Dictionary encodingMap = - new Dictionary() + internal static Dictionary encodingMap = + new Dictionary() { - { "255-254", FileSystemCmdletProviderEncoding.Unicode }, - { "254-255", FileSystemCmdletProviderEncoding.BigEndianUnicode }, - { "255-254-0-0", FileSystemCmdletProviderEncoding.UTF32 }, - { "0-0-254-255", FileSystemCmdletProviderEncoding.BigEndianUTF32 }, - { "239-187-191", FileSystemCmdletProviderEncoding.UTF8 }, + { "255-254", Encoding.Unicode }, + { "254-255", Encoding.BigEndianUnicode }, + { "255-254-0-0", Encoding.UTF32 }, + { "0-0-254-255", BigEndianUTF32Encoding }, + { "239-187-191", Encoding.UTF8 }, }; internal static char[] nonPrintableCharacters = { diff --git a/src/System.Management.Automation/engine/hostifaces/MshHostUserInterface.cs b/src/System.Management.Automation/engine/hostifaces/MshHostUserInterface.cs index d3cf16bc0a8..6d64065a5bd 100644 --- a/src/System.Management.Automation/engine/hostifaces/MshHostUserInterface.cs +++ b/src/System.Management.Automation/engine/hostifaces/MshHostUserInterface.cs @@ -1068,14 +1068,8 @@ internal string Path set { _path = value; - - Encoding = Encoding.UTF8; - FileSystemCmdletProviderEncoding fileEncoding = Utils.GetEncoding(value); - - if (fileEncoding != FileSystemCmdletProviderEncoding.Default) - { - Encoding = Utils.GetEncodingFromEnum(fileEncoding); - } + // Get the encoding from the file, or default (UTF8-NoBom) + Encoding = Utils.GetEncoding(value); } } private string _path; diff --git a/src/System.Management.Automation/namespaces/FileSystemProvider.cs b/src/System.Management.Automation/namespaces/FileSystemProvider.cs index 172e8849570..be489caa288 100644 --- a/src/System.Management.Automation/namespaces/FileSystemProvider.cs +++ b/src/System.Management.Automation/namespaces/FileSystemProvider.cs @@ -6619,7 +6619,7 @@ public IContentReader GetContentReader(string path) if (streamTypeSpecified) { - encoding = dynParams.EncodingType; + encoding = dynParams.Encoding; } // Get the wait value @@ -6770,7 +6770,7 @@ public IContentWriter GetContentWriter(string path) if (streamTypeSpecified) { - encoding = dynParams.EncodingType; + encoding = dynParams.Encoding; } #if !UNIX @@ -7477,73 +7477,6 @@ public static Hashtable Invoke(System.Management.Automation.PowerShell ps, FileS } } - /// - /// Defines the values that can be supplied as the encoding parameter in the - /// FileSystemContentDynamicParametersBase class. - /// - public enum FileSystemCmdletProviderEncoding - { - /// - /// No encoding. - /// - Unknown, - - /// - /// Unicode encoding. - /// - String, - - /// - /// Unicode encoding. - /// - Unicode, - - /// - /// Byte encoding. - /// - Byte, - - /// - /// Big Endian Unicode encoding. - /// - BigEndianUnicode, - - /// - /// UTF8 encoding. - /// - UTF8, - - /// - /// UTF7 encoding. - /// - UTF7, - - /// - /// UTF32 encoding. - /// - UTF32, - - /// - /// ASCII encoding. - /// - Ascii, - - /// - /// Default encoding. - /// - Default, - - /// - /// OEM encoding. - /// - Oem, - - /// - /// Big Endian UTF32 encoding. - /// - BigEndianUTF32, - } // FileSystemCmdletProviderEncoding - #endregion #region Dynamic Parameters @@ -7647,8 +7580,30 @@ public class FileSystemContentDynamicParametersBase /// reading data from the file. /// [Parameter] - public FileSystemCmdletProviderEncoding Encoding { get; set; } = FileSystemCmdletProviderEncoding.String; - + [ArgumentToEncodingTransformationAttribute()] + [ArgumentCompleter(typeof(EncodingArgumentCompleter))] + public Encoding Encoding + { + get + { + return _encoding; + } + set + { + if ( value == EncodingConversion.byteEncoding ) + { + _encoding = EncodingConversion.byteEncoding.ActualEncoding; + UsingByteEncoding = true; + } + else + { + _encoding = value; + } + // If an encoding was explicitly set, be sure to capture that. + WasStreamTypeSpecified = true; + } + } + private Encoding _encoding = ClrFacade.GetDefaultEncoding(); #if !UNIX /// /// A parameter to return a stream of an item. @@ -7657,28 +7612,16 @@ public class FileSystemContentDynamicParametersBase public String Stream { get; set; } #endif - /// - /// Gets the encoding from the specified StreamType parameter. - /// - public Encoding EncodingType - { - get - { - return Utils.GetEncodingFromEnum(Encoding); - } - } // EncodingType - /// /// Gets the Byte Encoding status of the StreamType parameter. Returns true /// if the stream was opened with "Byte" encoding, false otherwise. /// public bool UsingByteEncoding { - get - { - return Encoding == FileSystemCmdletProviderEncoding.Byte; - } // get + get { return _usingByteEncoding; } + set { _usingByteEncoding = value; } } // UsingByteEncoding + private bool _usingByteEncoding; /// /// Gets the status of the StreamType parameter. Returns true @@ -7688,9 +7631,14 @@ public bool WasStreamTypeSpecified { get { - return (Encoding != FileSystemCmdletProviderEncoding.String); - } // get + return _wasStreamTypeSpecified; + } + set + { + _wasStreamTypeSpecified = value; + } } // WasStreamTypeSpecified + private bool _wasStreamTypeSpecified; } // class FileSystemContentDynamicParametersBase diff --git a/src/System.Management.Automation/utils/ClrFacade.cs b/src/System.Management.Automation/utils/ClrFacade.cs index 03c4b252c88..399d561d6e1 100644 --- a/src/System.Management.Automation/utils/ClrFacade.cs +++ b/src/System.Management.Automation/utils/ClrFacade.cs @@ -93,14 +93,9 @@ internal static Encoding GetDefaultEncoding() { if (s_defaultEncoding == null) { -#if UNIX // PowerShell Core on Unix - s_defaultEncoding = new UTF8Encoding(false); -#else // PowerShell Core on Windows + // load all available encodings EncodingRegisterProvider(); - - uint currentAnsiCp = NativeMethods.GetACP(); - s_defaultEncoding = Encoding.GetEncoding((int)currentAnsiCp); -#endif + s_defaultEncoding = new UTF8Encoding(false); } return s_defaultEncoding; } @@ -109,16 +104,17 @@ internal static Encoding GetDefaultEncoding() /// /// Facade for getting OEM encoding + /// OEM encodings work on all platforms, or rather codepage 437 is available on both Windows /// internal static Encoding GetOEMEncoding() { if (s_oemEncoding == null) { -#if UNIX // PowerShell Core on Unix - s_oemEncoding = GetDefaultEncoding(); -#else // PowerShell Core on Windows + // load all available encodings EncodingRegisterProvider(); - +#if UNIX + s_oemEncoding = new UTF8Encoding(false); +#else uint oemCp = NativeMethods.GetOEMCP(); s_oemEncoding = Encoding.GetEncoding((int)oemCp); #endif diff --git a/src/System.Management.Automation/utils/PathUtils.cs b/src/System.Management.Automation/utils/PathUtils.cs index 632938e40e7..99d92e8bc24 100644 --- a/src/System.Management.Automation/utils/PathUtils.cs +++ b/src/System.Management.Automation/utils/PathUtils.cs @@ -188,6 +188,12 @@ internal static void ReportFileOpenFailure(Cmdlet cmdlet, string filePath, Excep cmdlet.ThrowTerminatingError(errorRecord); } + internal static StreamReader OpenStreamReader(PSCmdlet command, string filePath, Encoding encoding, bool isLiteralPath) + { + FileStream fileStream = OpenFileStream(filePath, command, isLiteralPath); + return new StreamReader(fileStream, encoding); + } + internal static StreamReader OpenStreamReader(PSCmdlet command, string filePath, string encoding, bool isLiteralPath) { FileStream fileStream = OpenFileStream(filePath, command, isLiteralPath); @@ -445,10 +451,23 @@ internal static class EncodingConversion internal const string BigEndianUnicode = "bigendianunicode"; internal const string Ascii = "ascii"; internal const string Utf8 = "utf8"; + internal const string Utf8NoBom = "utf8NoBOM"; + internal const string Utf8Bom = "utf8BOM"; internal const string Utf7 = "utf7"; internal const string Utf32 = "utf32"; internal const string Default = "default"; internal const string OEM = "oem"; + internal const string Byte = "byte"; // This will return unicode, but 'byte' is special as it uses unicode encoding without a BOM + internal static ByteEncoding byteEncoding = new ByteEncoding(); + // We will provide a partial list for tab completion we will convert to an encoding. + // If the user has an encoding object, we will cheerfully use it, but we can't use + // it as a validation set, as a user may have an specialty encoding which we have + // no idea about. + // We can't use Encoding.GetEncoding() as a validate set because we need to support + // unreal encodings like 'Byte' and 'OEM' + internal static readonly string[] TabCompletionResults = { + Ascii, BigEndianUnicode, Byte, OEM, Unicode, Utf7, Utf8, Utf8Bom, Utf8NoBom, Utf32 + }; /// /// retrieve the encoding parameter from the command line @@ -459,7 +478,7 @@ internal static Encoding Convert(Cmdlet cmdlet, string encoding) { if (string.IsNullOrEmpty(encoding)) { - // no parameter passed, default to Unicode (OS preferred) + // no parameter passed, default to (OS preferred) return System.Text.Encoding.Unicode; } @@ -470,6 +489,9 @@ internal static Encoding Convert(Cmdlet cmdlet, string encoding) if (string.Equals(encoding, String, StringComparison.OrdinalIgnoreCase)) return System.Text.Encoding.Unicode; + if (string.Equals(encoding, Byte, StringComparison.OrdinalIgnoreCase)) + return byteEncoding; + // these are the encodings the CLR supports if (string.Equals(encoding, Unicode, StringComparison.OrdinalIgnoreCase)) return System.Text.Encoding.Unicode; @@ -478,6 +500,12 @@ internal static Encoding Convert(Cmdlet cmdlet, string encoding) return System.Text.Encoding.BigEndianUnicode; if (string.Equals(encoding, Utf8, StringComparison.OrdinalIgnoreCase)) + return ClrFacade.GetDefaultEncoding(); + + if (string.Equals(encoding, Utf8NoBom, StringComparison.OrdinalIgnoreCase)) + return ClrFacade.GetDefaultEncoding(); + + if (string.Equals(encoding, Utf8Bom, StringComparison.OrdinalIgnoreCase)) return System.Text.Encoding.UTF8; if (string.Equals(encoding, Ascii, StringComparison.OrdinalIgnoreCase)) @@ -500,9 +528,8 @@ internal static Encoding Convert(Cmdlet cmdlet, string encoding) // error condition: unknown encoding value string validEncodingValues = string.Join( ", ", - new string[] { Unknown, String, Unicode, BigEndianUnicode, Ascii, Utf8, Utf7, Utf32, Default, OEM }); - string msg = StringUtil.Format(PathUtilsStrings.OutFile_WriteToFileEncodingUnknown, - encoding, validEncodingValues); + new string[] { Unknown, String, Unicode, BigEndianUnicode, Byte, Ascii, Utf8, Utf8Bom, Utf8NoBom, Utf7, Utf32, Default, OEM }); + string msg = StringUtil.Format(PathUtilsStrings.OutFile_WriteToFileEncodingUnknown, encoding, validEncodingValues); ErrorRecord errorRecord = new ErrorRecord( PSTraceSource.NewArgumentException("Encoding"), @@ -515,5 +542,104 @@ internal static Encoding Convert(Cmdlet cmdlet, string encoding) return null; } + + // We need a way to provide an encoding and also + // a way to notify that the user wants "byte" encoding + // which is just a stream + internal class ByteEncoding : System.Text.Encoding + { + // the encoding for this is not bigendian and not BOM + public Encoding ActualEncoding = new UnicodeEncoding(false, false); + public override unsafe int GetByteCount(char* c, int i1) { throw new NotImplementedException(); } + public override int GetByteCount(char[] c, int i1, int i2) { throw new NotImplementedException(); } + public override int GetByteCount(string s) { throw new NotImplementedException(); } + public override int GetByteCount(char[] c) { throw new NotImplementedException(); } + public override byte[] GetBytes(char[] c) { throw new NotImplementedException(); } + public override byte[] GetBytes(char[] c, int i1, int i2) { throw new NotImplementedException(); } + public override int GetBytes(char[] c, int i1, int i2, byte[] b, int i3) { throw new NotImplementedException(); } + public override byte[] GetBytes(string s) { throw new NotImplementedException(); } + public override unsafe int GetBytes(char* c, int i1, byte* b, int i2) { throw new NotImplementedException(); } + public override int GetBytes(string s, int i1, int i2, byte[] b, int i3) { throw new NotImplementedException(); } + public override int GetCharCount(byte[] b) { throw new NotImplementedException(); } + public override int GetCharCount(byte[] b, int i1, int i2) { throw new NotImplementedException(); } + public override unsafe int GetCharCount(byte* b, int i1) { throw new NotImplementedException(); } + public override char[] GetChars(byte[] b, int i1, int i2) { throw new NotImplementedException(); } + public override int GetChars(byte[] b, int i1, int i2, char[] c, int i3) { throw new NotImplementedException(); } + public override unsafe int GetChars(byte* b, int i1, char* c, int i2) { throw new NotImplementedException(); } + public override char[] GetChars(byte[] b) { throw new NotImplementedException(); } + public override Decoder GetDecoder() { throw new NotImplementedException(); } + public override Encoder GetEncoder() { throw new NotImplementedException(); } + public override int GetHashCode() { throw new NotImplementedException(); } + public override int GetMaxByteCount(int i1) { throw new NotImplementedException(); } + public override int GetMaxCharCount(int i1) { throw new NotImplementedException(); } + public override byte[] GetPreamble() { throw new NotImplementedException(); } + public override string GetString(byte[] b) { throw new NotImplementedException(); } + public override string GetString(byte[] b, int i1, int i2) { throw new NotImplementedException(); } + public override bool IsAlwaysNormalized(NormalizationForm f) { throw new NotImplementedException(); } + } + } + + /// + /// To make it easier to specify -Encoding parameter, we add an ArgumentTransformationAttribute here. + /// When the input data is of type string and is valid to be converted to System.Text.Encoding, we do + /// the conversion and return the converted value. Otherwise, we just return the input data. + /// + internal sealed class ArgumentToEncodingTransformationAttribute : ArgumentTransformationAttribute + { + public override object Transform(EngineIntrinsics engineIntrinsics, object inputData) + { + string encodingName; + if (LanguagePrimitives.TryConvertTo(inputData, out encodingName)) + { + if (string.Equals(encodingName, EncodingConversion.Unknown, StringComparison.OrdinalIgnoreCase) || + string.Equals(encodingName, EncodingConversion.Byte, StringComparison.OrdinalIgnoreCase) || + string.Equals(encodingName, EncodingConversion.String, StringComparison.OrdinalIgnoreCase) || + string.Equals(encodingName, EncodingConversion.Unicode, StringComparison.OrdinalIgnoreCase) || + string.Equals(encodingName, EncodingConversion.BigEndianUnicode, StringComparison.OrdinalIgnoreCase) || + string.Equals(encodingName, EncodingConversion.Utf8, StringComparison.OrdinalIgnoreCase) || + string.Equals(encodingName, EncodingConversion.Utf8Bom, StringComparison.OrdinalIgnoreCase) || + string.Equals(encodingName, EncodingConversion.Utf8NoBom, StringComparison.OrdinalIgnoreCase) || + string.Equals(encodingName, EncodingConversion.Utf7, StringComparison.OrdinalIgnoreCase) || + string.Equals(encodingName, EncodingConversion.Utf32, StringComparison.OrdinalIgnoreCase) || + string.Equals(encodingName, EncodingConversion.Ascii, StringComparison.OrdinalIgnoreCase) || + string.Equals(encodingName, EncodingConversion.Default, StringComparison.OrdinalIgnoreCase) || + string.Equals(encodingName, EncodingConversion.OEM, StringComparison.OrdinalIgnoreCase)) + { + // the encodingName is guaranteed to be valid, so it is safe to pass null to method + // Convert(Cmdlet cmdlet, string encoding) as the value of 'cmdlet'. + return EncodingConversion.Convert(null, encodingName); + } + } + return inputData; + } + + } + + internal class EncodingArgumentCompleter : IArgumentCompleter + { + /// + /// + /// + public IEnumerable CompleteArgument( + string commandName, + string parameterName, + string wordToComplete, + System.Management.Automation.Language.CommandAst commandAst, + System.Collections.IDictionary fakeBoundParameters) + { + List encodings = new List(); + foreach(string encoding in EncodingConversion.TabCompletionResults) + { + if (string.IsNullOrEmpty(wordToComplete)) + { + encodings.Add(new CompletionResult(encoding, encoding, CompletionResultType.Text, encoding)); + } + else if ( encoding.IndexOf(wordToComplete, StringComparison.InvariantCultureIgnoreCase) == 0 ) + { + encodings.Add(new CompletionResult(encoding, encoding, CompletionResultType.Text, encoding)); + } + } + return encodings; + } } } diff --git a/test/powershell/Language/Parser/RedirectionOperator.Tests.ps1 b/test/powershell/Language/Parser/RedirectionOperator.Tests.ps1 index e4e5fc7efcb..1e352999048 100644 --- a/test/powershell/Language/Parser/RedirectionOperator.Tests.ps1 +++ b/test/powershell/Language/Parser/RedirectionOperator.Tests.ps1 @@ -29,13 +29,14 @@ Describe "Redirection operator now supports encoding changes" -Tags "CI" { $psdefaultParameterValues.Remove("out-file:encoding") } - It "If encoding is unset, redirection should be Unicode" { + It "If encoding is unset, redirection should be UTF8 without bom" { $asciiString > TESTDRIVE:\file.txt $bytes = get-content -encoding byte TESTDRIVE:\file.txt - # create the expected - $BOM = [text.encoding]::unicode.GetPreamble() - $TXT = [text.encoding]::unicode.GetBytes($asciiString) - $CR = [text.encoding]::unicode.GetBytes($asciiCR) + # create the expected - utf8 encoding without a bom + $encoding = [text.utf8encoding]::new($false) + $BOM = $encoding.GetPreamble() + $TXT = $encoding.GetBytes($asciiString) + $CR = $encoding.GetBytes($asciiCR) $expectedBytes = .{ $BOM; $TXT; $CR } $bytes.Count | should be $expectedBytes.count for($i = 0; $i -lt $bytes.count; $i++) { diff --git a/test/powershell/Modules/Microsoft.PowerShell.Core/TestGetCommand.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Core/TestGetCommand.Tests.ps1 index 8f48d529620..d2d2c8f3784 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Core/TestGetCommand.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Core/TestGetCommand.Tests.ps1 @@ -211,7 +211,7 @@ $paramName = "Encoding" $results = get-command -verb get -noun content -Encoding Unicode VerifyDynamicParametersExist -cmdlet $results[0] -parameterNames $paramName - VerifyParameterType -cmdlet $results[0] -parameterName $paramName -parameterType Microsoft.PowerShell.Commands.FileSystemCmdletProviderEncoding + VerifyParameterType -cmdlet $results[0] -parameterName $paramName -parameterType System.Text.Encoding } It "Verify Single Cmdlet Using Verb&Noun ParameterSet With Usage" { @@ -261,7 +261,7 @@ $paramName = "Encoding" $results = Get-Command -verb get -noun content -encoding UTF8 VerifyDynamicParametersExist -cmdlet $results[0] -parameterNames $paramName - VerifyParameterType -cmdlet $results[0] -parameterName $paramName -ParameterType Microsoft.PowerShell.Commands.FileSystemCmdletProviderEncoding + VerifyParameterType -cmdlet $results[0] -parameterName $paramName -ParameterType System.Text.Encoding } #unsupported parameter: -synop @@ -269,6 +269,6 @@ $paramName = "Encoding" $results = get-command -verb get -noun content -encoding UTF8 -synop VerifyDynamicParametersExist -cmdlet $results[0] -parameterNames $paramName - VerifyParameterType -cmdlet $results[0] -parameterName $paramName -ParameterType Microsoft.PowerShell.Commands.FileSystemCmdletProviderEncoding + VerifyParameterType -cmdlet $results[0] -parameterName $paramName -ParameterType System.Text.Encoding } -} \ No newline at end of file +} diff --git a/test/powershell/Modules/Microsoft.PowerShell.Management/Get-Content.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Management/Get-Content.Tests.ps1 index d338a7cabea..1f68a3c8b86 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Management/Get-Content.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Management/Get-Content.Tests.ps1 @@ -99,13 +99,12 @@ baz "@ $expected = 'foo' $tailCount = 3 - [Microsoft.PowerShell.Commands.FileSystemCmdletProviderEncoding] $encoding = $EncodingName $testPath = Join-Path -Path $TestDrive -ChildPath 'TailWithEncoding.txt' - $content | Set-Content -Path $testPath -Encoding $encoding + $content | Set-Content -Path $testPath -Encoding $encodingName $expected = 'foo' - $actual = Get-Content -Path $testPath -Tail $tailCount -Encoding $encoding + $actual = Get-Content -Path $testPath -Tail $tailCount -Encoding $encodingName $actual.GetType() | Should Be "System.Object[]" $actual.Length | Should Be $tailCount $actual[0] | Should Be $expected diff --git a/test/powershell/engine/Basic/Encoding.Tests.ps1 b/test/powershell/engine/Basic/Encoding.Tests.ps1 new file mode 100644 index 00000000000..47fd757f93b --- /dev/null +++ b/test/powershell/engine/Basic/Encoding.Tests.ps1 @@ -0,0 +1,74 @@ +Describe "File encoding tests" -tag CI { + + Context "ParameterType for parameter 'Encoding' should be 'Encoding'" { + BeforeAll { + $testCases = get-command -module Microsoft.PowerShell.* | + Where-Object { $_.Parameters -and $_.Parameters['Encoding'] } | + ForEach-Object { @{ Command = $_ } } + } + It "Encoding parameter of command '' is type 'Encoding'" -testcase $testCases { + param ( $command ) + $command.Parameters['Encoding'].ParameterType.FullName | Should Be "System.Text.Encoding" + } + } + Context "File contents are UTF8 without BOM" { + BeforeAll { + $testStr = "t" + ([char]233) + "st" + $nl = [environment]::newline + $utf8Bytes = 116,195,169,115,116 + $nlBytes = [byte[]][char[]]$nl + $ExpectedWithNewline = $( $utf8Bytes ; $nlBytes ) + $outputFile = "${TESTDRIVE}/file.txt" + $utf8Preamble = [text.encoding]::UTF8.GetPreamble() + $simpleTestCases = @( + # new-item does not add CR/NL + @{ Command = "new-item"; Parameters = @{ type = "file";value = $testStr; Path = $outputFile }; Expected = $utf8Bytes ; Operator = "be" } + # the following commands add a CR/NL + @{ Command = "set-content"; Parameters = @{ value = $testStr; Path = $outputFile }; Expected = $ExpectedWithNewline ; Operator = "be" } + @{ Command = "add-content"; Parameters = @{ value = $testStr; Path = $outputFile }; Expected = $ExpectedWithNewline ; Operator = "be" } + @{ Command = "out-file"; Parameters = @{ InputObject = $testStr; Path = $outputFile }; Expected = $ExpectedWithNewline ; Operator = "be" } + # Redirection + @{ Command = { $testStr > $outputFile } ; Expected = $ExpectedWithNewline ; Operator = "be" } + ) + function Get-FileBytes ( $path ) { + [io.file]::ReadAllBytes($path) + } + } + + AfterEach { + if ( test-path $outputFile ) { + remove-item -force $outputFile + } + } + + It " produces correct content ''" -testcases $simpleTestCases { + param ( $Command, $parameters, $Expected, $Operator) + & $command @parameters + $bytes = Get-FileBytes $outputFile + $bytes -join "-" | should ${Operator} ($Expected -join "-") + } + + It "Export-CSV creates file with UTF-8 encoding without BOM" { + [pscustomobject]@{ Key = $testStr } | export-csv $outputFile + $bytes = Get-FileBytes $outputFile + $bytes[0,1,2] -join "-" | should not be ($utf8Preamble -join "-") + $bytes -join "-" | should match ($utf8bytes -join "-") + } + + It "Export-CliXml creates file with UTF-8 encoding without BOM" { + [pscustomobject]@{ Key = $testStr } | export-clixml $outputFile + $bytes = Get-FileBytes $outputFile + $bytes[0,1,2] -join "-" | should not be ($utf8Preamble -join "-") + $bytes -join "-" | should match ($utf8bytes -join "-") + } + + It "Appends correctly on non-Windows systems" -skip:$IsWindows { + bash -c "echo '${testStr}' > $outputFile" + ${testStr} >> $outputFile + $bytes = Get-FileBytes $outputFile + $Expected = $( $ExpectedWithNewline; $ExpectedWithNewline ) + $bytes -join "-" | should be ($Expected -join "-") + } + + } +} From 431370ab7f64b5f21aef596bde192369281800e3 Mon Sep 17 00:00:00 2001 From: James Truher Date: Thu, 12 Oct 2017 15:38:06 -0700 Subject: [PATCH 2/8] Update to follow coding style guidelines --- .../engine/Utils.cs | 2 +- .../namespaces/FileSystemProvider.cs | 20 ++----------------- .../utils/ClrFacade.cs | 2 +- .../utils/PathUtils.cs | 2 +- 4 files changed, 5 insertions(+), 21 deletions(-) diff --git a/src/System.Management.Automation/engine/Utils.cs b/src/System.Management.Automation/engine/Utils.cs index 61a8baab95d..f6b6449c9d8 100644 --- a/src/System.Management.Automation/engine/Utils.cs +++ b/src/System.Management.Automation/engine/Utils.cs @@ -1219,7 +1219,7 @@ internal static Encoding GetEncoding(string path) // BigEndianUTF32 encoding is possible, but requires creation - internal static Encoding BigEndianUTF32Encoding = new UTF32Encoding(true,true); + internal static Encoding BigEndianUTF32Encoding = new UTF32Encoding(bigEndian: true, byteOrderMark: true); // [System.Text.Encoding]::GetEncodings() | Where-Object { $_.GetEncoding().GetPreamble() } | // Add-Member ScriptProperty Preamble { $this.GetEncoding().GetPreamble() -join "-" } -PassThru | // Format-Table -Auto diff --git a/src/System.Management.Automation/namespaces/FileSystemProvider.cs b/src/System.Management.Automation/namespaces/FileSystemProvider.cs index be489caa288..b193ae4c303 100644 --- a/src/System.Management.Automation/namespaces/FileSystemProvider.cs +++ b/src/System.Management.Automation/namespaces/FileSystemProvider.cs @@ -7616,29 +7616,13 @@ public Encoding Encoding /// Gets the Byte Encoding status of the StreamType parameter. Returns true /// if the stream was opened with "Byte" encoding, false otherwise. /// - public bool UsingByteEncoding - { - get { return _usingByteEncoding; } - set { _usingByteEncoding = value; } - } // UsingByteEncoding - private bool _usingByteEncoding; + public bool UsingByteEncoding { get; set; } /// /// Gets the status of the StreamType parameter. Returns true /// if the stream was opened with a user-specified encoding, false otherwise. /// - public bool WasStreamTypeSpecified - { - get - { - return _wasStreamTypeSpecified; - } - set - { - _wasStreamTypeSpecified = value; - } - } // WasStreamTypeSpecified - private bool _wasStreamTypeSpecified; + public bool WasStreamTypeSpecified { get; set; } } // class FileSystemContentDynamicParametersBase diff --git a/src/System.Management.Automation/utils/ClrFacade.cs b/src/System.Management.Automation/utils/ClrFacade.cs index 399d561d6e1..f8eafa3622e 100644 --- a/src/System.Management.Automation/utils/ClrFacade.cs +++ b/src/System.Management.Automation/utils/ClrFacade.cs @@ -104,7 +104,7 @@ internal static Encoding GetDefaultEncoding() /// /// Facade for getting OEM encoding - /// OEM encodings work on all platforms, or rather codepage 437 is available on both Windows + /// OEM encodings work on all platforms, or rather codepage 437 is available on both Windows and Non-Windows /// internal static Encoding GetOEMEncoding() { diff --git a/src/System.Management.Automation/utils/PathUtils.cs b/src/System.Management.Automation/utils/PathUtils.cs index 99d92e8bc24..9ac8cb703a8 100644 --- a/src/System.Management.Automation/utils/PathUtils.cs +++ b/src/System.Management.Automation/utils/PathUtils.cs @@ -549,7 +549,7 @@ internal static Encoding Convert(Cmdlet cmdlet, string encoding) internal class ByteEncoding : System.Text.Encoding { // the encoding for this is not bigendian and not BOM - public Encoding ActualEncoding = new UnicodeEncoding(false, false); + public Encoding ActualEncoding = new UnicodeEncoding(bigEndian: false, byteOrderMark: false); public override unsafe int GetByteCount(char* c, int i1) { throw new NotImplementedException(); } public override int GetByteCount(char[] c, int i1, int i2) { throw new NotImplementedException(); } public override int GetByteCount(string s) { throw new NotImplementedException(); } From 53b31e91d4a58cf82e817d433c1ae849fd957ccd Mon Sep 17 00:00:00 2001 From: James Truher Date: Thu, 12 Oct 2017 15:41:48 -0700 Subject: [PATCH 3/8] Fix cmdlet casing to follow guidelines --- .../TestGetCommand.Tests.ps1 | 40 +++++++++---------- .../engine/Basic/Encoding.Tests.ps1 | 35 ++++++++-------- 2 files changed, 37 insertions(+), 38 deletions(-) diff --git a/test/powershell/Modules/Microsoft.PowerShell.Core/TestGetCommand.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Core/TestGetCommand.Tests.ps1 index d2d2c8f3784..11b0e6f7841 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Core/TestGetCommand.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Core/TestGetCommand.Tests.ps1 @@ -13,11 +13,11 @@ ) DynamicParam { - if ( ! $testToRun ) { - $testToRun = "returnnull" + if ( ! $TestToRun ) { + $TestToRun = "returnnull" } $dynamicParamDictionary = [System.Management.Automation.RuntimeDefinedParameterDictionary]::new() - switch ( $testToRun ) + switch ( $TestToRun ) { "returnnull" { $dynamicParamDictionary = $null @@ -161,16 +161,16 @@ } } - It "Verify that get-command get-content includes the dynamic parameters when the cmdlet is checked against the file system provider implementation" { + It "Verify that Get-Command Get-Content includes the dynamic parameters when the cmdlet is checked against the file system provider implementation" { $fullPath = Join-Path $TestDrive -ChildPath "blah" New-Item -Path $fullPath -ItemType directory -Force - $results = get-command get-content -path $fullPath + $results = Get-Command Get-Content -Path $fullPath $dynamicParameter = "Wait", "Encoding", "Delimiter" VerifyDynamicParametersExist -cmdlet $results[0] -parameterNames $dynamicParameter } - It "Verify that get-command get-content doesn't have any dynamic parameters for Function provider" { - $results =get-command get-content -path function: + It "Verify that Get-Command Get-Content doesn't have any dynamic parameters for Function provider" { + $results =Get-Command Get-Content -Path function: $dynamicParameter = "Wait", "Encoding", "Delimiter" foreach ($dynamicPara in $dynamicParameter) { @@ -179,14 +179,14 @@ } It "Verify that the specified dynamic parameter exists in the CmdletInfo result returned" { - $results = get-command testgetcommand-dynamicparametersdcr -testtorun return1 + $results = Get-Command TestGetCommand-DynamicParametersDCR -TestToRun return1 $dynamicParameter = "OneString" VerifyDynamicParametersExist -cmdlet $results[0] -parameterNames $dynamicParameter VerifyParameterType -cmdlet $results[0] -parameterName $dynamicParameter -ParameterType string } It "Verify three dynamic parameters are created properly" { - $results = get-command testgetcommand-dynamicparametersdcr -testtorun return3 + $results = Get-Command TestGetCommand-DynamicParametersDCR -TestToRun return3 $dynamicParameter = "OneString", "TwoInt", "ThreeBool" VerifyDynamicParametersExist -cmdlet $results[0] -parameterNames $dynamicParameter @@ -196,26 +196,26 @@ } It "Verify dynamic parameter type is process" { - $results = get-command testgetcommand-dynamicparametersdcr -args '-testtorun','returngenericparameter','-parametertype','System.Diagnostics.Process' + $results = Get-Command TestGetCommand-DynamicParametersDCR -Args '-TestToRun','returngenericparameter','-parametertype','System.Diagnostics.Process' VerifyParameterType -cmdlet $results[0] -parameterName "TypedValue" -parameterType System.Diagnostics.Process } It "Verify a single cmdlet returned using verb and noun parameter set syntax works properly" { $paramName = "OneString" - $results = get-command -verb testgetcommand -noun dynamicparametersdcr -testtorun Return1 + $results = Get-Command -Verb TestGetCommand -Noun DynamicParametersDCR -TestToRun Return1 VerifyDynamicParametersExist -cmdlet $results[0] -parameterNames $paramName VerifyParameterType -cmdlet $results[0] -parameterName $paramName -parameterType string } It "Verify Single Cmdlet Using Verb&Noun ParameterSet" { $paramName = "Encoding" - $results = get-command -verb get -noun content -Encoding Unicode + $results = Get-Command -Verb get -Noun content -Encoding Unicode VerifyDynamicParametersExist -cmdlet $results[0] -parameterNames $paramName VerifyParameterType -cmdlet $results[0] -parameterName $paramName -parameterType System.Text.Encoding } It "Verify Single Cmdlet Using Verb&Noun ParameterSet With Usage" { - $results = get-command -verb get -noun content -Encoding Unicode -syntax + $results = Get-Command -Verb get -Noun content -Encoding Unicode -Syntax $results.ToString() | Should Match "-Encoding" $results.ToString() | Should Match "-Wait" $results.ToString() | Should Match "-Delimiter" @@ -226,24 +226,24 @@ $tempFile = "mytempfile.ps1" $fullPath = Join-Path $TestDrive -ChildPath $tempFile "$a = dir" > $fullPath - $results = get-command $fullPath + $results = Get-Command $fullPath $results.Name | Should Be $tempFile $results.Definition | Should Be $fullPath } It "Two dynamic parameters are created properly" { - $results = get-command testgetcommand-dynamicparametersdcr -testtorun return2 + $results = Get-Command TestGetCommand-DynamicParametersDCR -TestToRun return2 $dynamicParameter = "OneString", "TwoInt" VerifyDynamicParametersExist -cmdlet $results[0] -parameterNames $dynamicParameter VerifyParameterType -cmdlet $results[0] -parameterName "OneString" -ParameterType string VerifyParameterType -cmdlet $results[0] -parameterName "TwoInt" -ParameterType int } - It "Throw an Exception when set testtorun to 'returnduplicateparameter'" { + It "Throw an Exception when set TestToRun to 'returnduplicateparameter'" { try { - Get-Command testgetcommand-dynamicparametersdcr -testtorun returnduplicateparameter -ErrorAction Stop + Get-Command TestGetCommand-DynamicParametersDCR -TestToRun returnduplicateparameter -ErrorAction Stop throw "No Exception!" } catch @@ -253,13 +253,13 @@ } It "verify if get the proper dynamic parameter type skipped by issue #1430" -Pending { - $results = Get-Command testgetcommand-dynamicparametersdcr -testtorun returngenericparameter -parametertype System.Diagnostics.Process + $results = Get-Command TestGetCommand-DynamicParametersDCR -TestToRun returngenericparameter -parametertype System.Diagnostics.Process VerifyParameterType -cmdlet $results[0] -parameterName "TypedValue" -parameterType System.Diagnostics.Process } It "It works with Single Cmdlet Using Verb&Noun ParameterSet" { $paramName = "Encoding" - $results = Get-Command -verb get -noun content -encoding UTF8 + $results = Get-Command -Verb get -Noun content -encoding UTF8 VerifyDynamicParametersExist -cmdlet $results[0] -parameterNames $paramName VerifyParameterType -cmdlet $results[0] -parameterName $paramName -ParameterType System.Text.Encoding } @@ -267,7 +267,7 @@ #unsupported parameter: -synop It "[Unsupported]It works with Single Cmdlet Using Verb&Noun ParameterSet With Synopsis" -Pending { $paramName = "Encoding" - $results = get-command -verb get -noun content -encoding UTF8 -synop + $results = Get-Command -Verb get -Noun content -encoding UTF8 -Synop VerifyDynamicParametersExist -cmdlet $results[0] -parameterNames $paramName VerifyParameterType -cmdlet $results[0] -parameterName $paramName -ParameterType System.Text.Encoding } diff --git a/test/powershell/engine/Basic/Encoding.Tests.ps1 b/test/powershell/engine/Basic/Encoding.Tests.ps1 index 47fd757f93b..309f2c26af1 100644 --- a/test/powershell/engine/Basic/Encoding.Tests.ps1 +++ b/test/powershell/engine/Basic/Encoding.Tests.ps1 @@ -1,32 +1,32 @@ -Describe "File encoding tests" -tag CI { +Describe "File encoding tests" -Tag CI { Context "ParameterType for parameter 'Encoding' should be 'Encoding'" { BeforeAll { - $testCases = get-command -module Microsoft.PowerShell.* | + $testCases = Get-Command -Module Microsoft.PowerShell.* | Where-Object { $_.Parameters -and $_.Parameters['Encoding'] } | ForEach-Object { @{ Command = $_ } } } - It "Encoding parameter of command '' is type 'Encoding'" -testcase $testCases { + It "Encoding parameter of command '' is type 'Encoding'" -Testcase $testCases { param ( $command ) - $command.Parameters['Encoding'].ParameterType.FullName | Should Be "System.Text.Encoding" + $command.Parameters['Encoding'].ParameterType.FullName | Should BeExactly "System.Text.Encoding" } } Context "File contents are UTF8 without BOM" { BeforeAll { $testStr = "t" + ([char]233) + "st" $nl = [environment]::newline - $utf8Bytes = 116,195,169,115,116 + $utf8Bytes = 116,195,169,115,116 $nlBytes = [byte[]][char[]]$nl $ExpectedWithNewline = $( $utf8Bytes ; $nlBytes ) $outputFile = "${TESTDRIVE}/file.txt" $utf8Preamble = [text.encoding]::UTF8.GetPreamble() $simpleTestCases = @( - # new-item does not add CR/NL - @{ Command = "new-item"; Parameters = @{ type = "file";value = $testStr; Path = $outputFile }; Expected = $utf8Bytes ; Operator = "be" } + # New-Item does not add CR/NL + @{ Command = "New-Item"; Parameters = @{ type = "file";value = $testStr; Path = $outputFile }; Expected = $utf8Bytes ; Operator = "be" } # the following commands add a CR/NL - @{ Command = "set-content"; Parameters = @{ value = $testStr; Path = $outputFile }; Expected = $ExpectedWithNewline ; Operator = "be" } - @{ Command = "add-content"; Parameters = @{ value = $testStr; Path = $outputFile }; Expected = $ExpectedWithNewline ; Operator = "be" } - @{ Command = "out-file"; Parameters = @{ InputObject = $testStr; Path = $outputFile }; Expected = $ExpectedWithNewline ; Operator = "be" } + @{ Command = "Set-Content"; Parameters = @{ value = $testStr; Path = $outputFile }; Expected = $ExpectedWithNewline ; Operator = "be" } + @{ Command = "Add-Content"; Parameters = @{ value = $testStr; Path = $outputFile }; Expected = $ExpectedWithNewline ; Operator = "be" } + @{ Command = "Out-File"; Parameters = @{ InputObject = $testStr; Path = $outputFile }; Expected = $ExpectedWithNewline ; Operator = "be" } # Redirection @{ Command = { $testStr > $outputFile } ; Expected = $ExpectedWithNewline ; Operator = "be" } ) @@ -35,13 +35,13 @@ Describe "File encoding tests" -tag CI { } } - AfterEach { - if ( test-path $outputFile ) { - remove-item -force $outputFile + AfterEach { + if ( Test-Path $outputFile ) { + Remove-Item -Force $outputFile } } - It " produces correct content ''" -testcases $simpleTestCases { + It " produces correct content ''" -Testcases $simpleTestCases { param ( $Command, $parameters, $Expected, $Operator) & $command @parameters $bytes = Get-FileBytes $outputFile @@ -49,26 +49,25 @@ Describe "File encoding tests" -tag CI { } It "Export-CSV creates file with UTF-8 encoding without BOM" { - [pscustomobject]@{ Key = $testStr } | export-csv $outputFile + [pscustomobject]@{ Key = $testStr } | Export-Csv $outputFile $bytes = Get-FileBytes $outputFile $bytes[0,1,2] -join "-" | should not be ($utf8Preamble -join "-") $bytes -join "-" | should match ($utf8bytes -join "-") } It "Export-CliXml creates file with UTF-8 encoding without BOM" { - [pscustomobject]@{ Key = $testStr } | export-clixml $outputFile + [pscustomobject]@{ Key = $testStr } | Export-Clixml $outputFile $bytes = Get-FileBytes $outputFile $bytes[0,1,2] -join "-" | should not be ($utf8Preamble -join "-") $bytes -join "-" | should match ($utf8bytes -join "-") } - It "Appends correctly on non-Windows systems" -skip:$IsWindows { + It "Appends correctly on non-Windows systems" -Skip:$IsWindows { bash -c "echo '${testStr}' > $outputFile" ${testStr} >> $outputFile $bytes = Get-FileBytes $outputFile $Expected = $( $ExpectedWithNewline; $ExpectedWithNewline ) $bytes -join "-" | should be ($Expected -join "-") } - } } From 6ef85800e3246446b24ed5fe8ca768a26f5c642f Mon Sep 17 00:00:00 2001 From: James Truher Date: Fri, 13 Oct 2017 10:22:11 -0700 Subject: [PATCH 4/8] Change the encoding to utf8 when the converter gets an empty string refactor encoding class and other encoding utilities into new file update redirection operator tests to convert string matching for encoding into dictionary, which should make it more efficient --- .../utils/EncodingUtils.cs | 188 ++++++++++++++++ .../utils/PathUtils.cs | 200 ------------------ .../Parser/RedirectionOperator.Tests.ps1 | 28 +-- 3 files changed, 202 insertions(+), 214 deletions(-) create mode 100644 src/System.Management.Automation/utils/EncodingUtils.cs diff --git a/src/System.Management.Automation/utils/EncodingUtils.cs b/src/System.Management.Automation/utils/EncodingUtils.cs new file mode 100644 index 00000000000..bae66e7820e --- /dev/null +++ b/src/System.Management.Automation/utils/EncodingUtils.cs @@ -0,0 +1,188 @@ +/********************************************************************++ +Copyright (c) Microsoft Corporation. All rights reserved. +--********************************************************************/ + +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Text; + +using System.Management.Automation.Internal; + +namespace System.Management.Automation +{ + + internal static class EncodingConversion + { + internal const string Unknown = "unknown"; + internal const string String = "string"; + internal const string Unicode = "unicode"; + internal const string BigEndianUnicode = "bigendianunicode"; + internal const string Ascii = "ascii"; + internal const string Utf8 = "utf8"; + internal const string Utf8NoBom = "utf8NoBOM"; + internal const string Utf8Bom = "utf8BOM"; + internal const string Utf7 = "utf7"; + internal const string Utf32 = "utf32"; + internal const string Default = "default"; + internal const string OEM = "oem"; + internal const string Byte = "byte"; // This will return unicode, but 'byte' is special as it uses unicode encoding without a BOM + // We're using this custom ByteEncoding so the *-Content cmdlets can provide `byte` as an encoding, which has the + // behavior of [io.file]::ReadAllBytes(). When we changed the type of the parameter to Encoding + // and created the ArgumentTransformationAttribute to convert strings to encodings to be more usable, it means that + // we have to check to se + internal static readonly ByteEncoding byteEncoding = new ByteEncoding(); + // We will provide a partial list for tab completion we will convert to an encoding. + // If the user has an encoding object, we will cheerfully use it, but we can't use + // it as a validation set, as a user may have an specialty encoding which we have + // no idea about. + // We can't use Encoding.GetEncoding() as a validate set because we need to support + // unreal encodings like 'Byte' and 'OEM' + internal static readonly string[] TabCompletionResults = { + Ascii, BigEndianUnicode, Byte, OEM, Unicode, Utf7, Utf8, Utf8Bom, Utf8NoBom, Utf32 + }; + + internal static Dictionary encodingMap = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + { Ascii, System.Text.Encoding.ASCII }, + { BigEndianUnicode, System.Text.Encoding.BigEndianUnicode }, + { Byte, byteEncoding }, + { Default, ClrFacade.GetDefaultEncoding() }, + { OEM, ClrFacade.GetOEMEncoding() }, + { Unicode, System.Text.Encoding.Unicode }, + { Utf7, System.Text.Encoding.UTF7 }, + { Utf8, ClrFacade.GetDefaultEncoding() }, + { Utf8Bom, System.Text.Encoding.UTF8 }, + { Utf8NoBom, ClrFacade.GetDefaultEncoding() }, + { Utf32, System.Text.Encoding.UTF32 }, + { String, System.Text.Encoding.Unicode }, + { Unknown, System.Text.Encoding.Unicode }, + }; + + /// + /// retrieve the encoding parameter from the command line + /// it throws if the encoding does not match the known ones + /// + /// a System.Text.Encoding object (null if no encoding specified) + internal static Encoding Convert(Cmdlet cmdlet, string encoding) + { + if (string.IsNullOrEmpty(encoding)) + { + // no parameter passed, default to UTF8 + return ClrFacade.GetDefaultEncoding(); + } + + Encoding foundEncoding; + if (encodingMap.TryGetValue(encoding, out foundEncoding)) + { + return foundEncoding; + } + + // error condition: unknown encoding value + string validEncodingValues = string.Join(", ", TabCompletionResults); + string msg = StringUtil.Format(PathUtilsStrings.OutFile_WriteToFileEncodingUnknown, encoding, validEncodingValues); + + ErrorRecord errorRecord = new ErrorRecord( + PSTraceSource.NewArgumentException("Encoding"), + "WriteToFileEncodingUnknown", + ErrorCategory.InvalidArgument, + null); + + errorRecord.ErrorDetails = new ErrorDetails(msg); + cmdlet.ThrowTerminatingError(errorRecord); + + return null; + } + + // We need a way to provide an encoding and also + // a way to notify that the user wants "byte" encoding which is just a stream of bytes + // Because we've changed the parameter type to Encoding, we need a way + // to continue to support the *-Content cmdlets use of -Encoding Byte, so we need + // to have the ArgumentTransformationAttribute actually return an encoding + // moreover, we need a way to notify the -Content cmdlets that the user requested `byte` + // so we can't just return the actual encoding we want, thus this artificial encoding + internal class ByteEncoding : System.Text.Encoding + { + // the encoding for this is not bigendian and not BOM + public Encoding ActualEncoding = new UnicodeEncoding(bigEndian: false, byteOrderMark: false); + // redirect all the calls to the ActualEncoding + public override unsafe int GetByteCount(char* chars, int count) { return ActualEncoding.GetByteCount(chars, count); } + public override int GetByteCount(char[] chars, int index, int count) { return ActualEncoding.GetByteCount(chars, index, count); } + public override int GetByteCount(string s) { return ActualEncoding.GetByteCount(s); } + public override int GetByteCount(char[] chars) { return ActualEncoding.GetByteCount(chars); } + public override byte[] GetBytes(char[] chars) { return ActualEncoding.GetBytes(chars); } + public override byte[] GetBytes(char[] chars, int index, int count) { return ActualEncoding.GetBytes(chars, index, count); } + public override int GetBytes(char[] chars, int charIndex, int charCount, byte[] bytes, int byteIndex) { return ActualEncoding.GetBytes(chars, charIndex, charCount, bytes, byteIndex); } + public override byte[] GetBytes(string s) { return ActualEncoding.GetBytes(s); } + public override unsafe int GetBytes(char* chars, int charCount, byte* bytes, int byteCount) { return ActualEncoding.GetBytes(chars, charCount, bytes, byteCount); } + public override int GetBytes(string s, int charIndex, int charCount, byte[] bytes, int byteIndex) { return ActualEncoding.GetBytes(s, charIndex, charCount, bytes, byteIndex); } + public override int GetCharCount(byte[] bytes) { return ActualEncoding.GetCharCount(bytes); } + public override int GetCharCount(byte[] bytes, int index, int count) { return ActualEncoding.GetCharCount(bytes, index, count); } + public override unsafe int GetCharCount(byte* bytes, int count) { return ActualEncoding.GetCharCount(bytes, count); } + public override char[] GetChars(byte[] bytes, int index, int count) { return ActualEncoding.GetChars(bytes, index, count); } + public override int GetChars(byte[] bytes, int byteIndex, int byteCount, char[] chars, int charIndex) { return ActualEncoding.GetChars(bytes, byteIndex, byteCount, chars, charIndex); } + public override unsafe int GetChars(byte* bytes, int byteCount, char* chars, int charCount) { return ActualEncoding.GetChars(bytes, byteCount, chars, charCount); } + public override char[] GetChars(byte[] bytes) { return ActualEncoding.GetChars(bytes); } + public override Decoder GetDecoder() { return ActualEncoding.GetDecoder(); } + public override Encoder GetEncoder() { return ActualEncoding.GetEncoder(); } + public override int GetHashCode() { return ActualEncoding.GetHashCode(); } + public override int GetMaxByteCount(int charCount) { return ActualEncoding.GetMaxByteCount(charCount); } + public override int GetMaxCharCount(int byteCount) { return ActualEncoding.GetMaxCharCount(byteCount); } + public override byte[] GetPreamble() { return ActualEncoding.GetPreamble(); } + public override string GetString(byte[] bytes) { return ActualEncoding.GetString(bytes); } + public override string GetString(byte[] bytes, int index, int count) { return ActualEncoding.GetString(bytes, index, count); } + public override bool IsAlwaysNormalized(NormalizationForm normalizationForm) { return ActualEncoding.IsAlwaysNormalized(normalizationForm); } + } + } + + /// + /// To make it easier to specify -Encoding parameter, we add an ArgumentTransformationAttribute here. + /// When the input data is of type string and is valid to be converted to System.Text.Encoding, we do + /// the conversion and return the converted value. Otherwise, we just return the input data. + /// + internal sealed class ArgumentToEncodingTransformationAttribute : ArgumentTransformationAttribute + { + public override object Transform(EngineIntrinsics engineIntrinsics, object inputData) + { + string encodingName; + if (LanguagePrimitives.TryConvertTo(inputData, out encodingName)) + { + Encoding foundEncoding; + if (EncodingConversion.encodingMap.TryGetValue(encodingName, out foundEncoding)) + { + return foundEncoding; + } + } + return inputData; + } + + } + + internal class EncodingArgumentCompleter : IArgumentCompleter + { + /// + /// + /// + public IEnumerable CompleteArgument( + string commandName, + string parameterName, + string wordToComplete, + System.Management.Automation.Language.CommandAst commandAst, + System.Collections.IDictionary fakeBoundParameters) + { + List encodings = new List(); + foreach(string encoding in EncodingConversion.TabCompletionResults) + { + if (string.IsNullOrEmpty(wordToComplete)) + { + encodings.Add(new CompletionResult(encoding, encoding, CompletionResultType.Text, encoding)); + } + else if (encoding.IndexOf(wordToComplete, StringComparison.InvariantCultureIgnoreCase) == 0) + { + encodings.Add(new CompletionResult(encoding, encoding, CompletionResultType.Text, encoding)); + } + } + return encodings; + } + } +} diff --git a/src/System.Management.Automation/utils/PathUtils.cs b/src/System.Management.Automation/utils/PathUtils.cs index 9ac8cb703a8..5d6879ea750 100644 --- a/src/System.Management.Automation/utils/PathUtils.cs +++ b/src/System.Management.Automation/utils/PathUtils.cs @@ -442,204 +442,4 @@ internal static DirectoryInfo CreateTemporaryDirectory() return new DirectoryInfo(moduleDirectory.FullName); } } - - internal static class EncodingConversion - { - internal const string Unknown = "unknown"; - internal const string String = "string"; - internal const string Unicode = "unicode"; - internal const string BigEndianUnicode = "bigendianunicode"; - internal const string Ascii = "ascii"; - internal const string Utf8 = "utf8"; - internal const string Utf8NoBom = "utf8NoBOM"; - internal const string Utf8Bom = "utf8BOM"; - internal const string Utf7 = "utf7"; - internal const string Utf32 = "utf32"; - internal const string Default = "default"; - internal const string OEM = "oem"; - internal const string Byte = "byte"; // This will return unicode, but 'byte' is special as it uses unicode encoding without a BOM - internal static ByteEncoding byteEncoding = new ByteEncoding(); - // We will provide a partial list for tab completion we will convert to an encoding. - // If the user has an encoding object, we will cheerfully use it, but we can't use - // it as a validation set, as a user may have an specialty encoding which we have - // no idea about. - // We can't use Encoding.GetEncoding() as a validate set because we need to support - // unreal encodings like 'Byte' and 'OEM' - internal static readonly string[] TabCompletionResults = { - Ascii, BigEndianUnicode, Byte, OEM, Unicode, Utf7, Utf8, Utf8Bom, Utf8NoBom, Utf32 - }; - - /// - /// retrieve the encoding parameter from the command line - /// it throws if the encoding does not match the known ones - /// - /// a System.Text.Encoding object (null if no encoding specified) - internal static Encoding Convert(Cmdlet cmdlet, string encoding) - { - if (string.IsNullOrEmpty(encoding)) - { - // no parameter passed, default to (OS preferred) - return System.Text.Encoding.Unicode; - } - - // Default to unicode (this matches Get-Content) - if (string.Equals(encoding, Unknown, StringComparison.OrdinalIgnoreCase)) - return System.Text.Encoding.Unicode; - - if (string.Equals(encoding, String, StringComparison.OrdinalIgnoreCase)) - return System.Text.Encoding.Unicode; - - if (string.Equals(encoding, Byte, StringComparison.OrdinalIgnoreCase)) - return byteEncoding; - - // these are the encodings the CLR supports - if (string.Equals(encoding, Unicode, StringComparison.OrdinalIgnoreCase)) - return System.Text.Encoding.Unicode; - - if (string.Equals(encoding, BigEndianUnicode, StringComparison.OrdinalIgnoreCase)) - return System.Text.Encoding.BigEndianUnicode; - - if (string.Equals(encoding, Utf8, StringComparison.OrdinalIgnoreCase)) - return ClrFacade.GetDefaultEncoding(); - - if (string.Equals(encoding, Utf8NoBom, StringComparison.OrdinalIgnoreCase)) - return ClrFacade.GetDefaultEncoding(); - - if (string.Equals(encoding, Utf8Bom, StringComparison.OrdinalIgnoreCase)) - return System.Text.Encoding.UTF8; - - if (string.Equals(encoding, Ascii, StringComparison.OrdinalIgnoreCase)) - return System.Text.Encoding.ASCII; - - if (string.Equals(encoding, Utf7, StringComparison.OrdinalIgnoreCase)) - return System.Text.Encoding.UTF7; - - if (string.Equals(encoding, Utf32, StringComparison.OrdinalIgnoreCase)) - return System.Text.Encoding.UTF32; - - if (string.Equals(encoding, Default, StringComparison.OrdinalIgnoreCase)) - return ClrFacade.GetDefaultEncoding(); - - if (string.Equals(encoding, OEM, StringComparison.OrdinalIgnoreCase)) - { - return ClrFacade.GetOEMEncoding(); - } - - // error condition: unknown encoding value - string validEncodingValues = string.Join( - ", ", - new string[] { Unknown, String, Unicode, BigEndianUnicode, Byte, Ascii, Utf8, Utf8Bom, Utf8NoBom, Utf7, Utf32, Default, OEM }); - string msg = StringUtil.Format(PathUtilsStrings.OutFile_WriteToFileEncodingUnknown, encoding, validEncodingValues); - - ErrorRecord errorRecord = new ErrorRecord( - PSTraceSource.NewArgumentException("Encoding"), - "WriteToFileEncodingUnknown", - ErrorCategory.InvalidArgument, - null); - - errorRecord.ErrorDetails = new ErrorDetails(msg); - cmdlet.ThrowTerminatingError(errorRecord); - - return null; - } - - // We need a way to provide an encoding and also - // a way to notify that the user wants "byte" encoding - // which is just a stream - internal class ByteEncoding : System.Text.Encoding - { - // the encoding for this is not bigendian and not BOM - public Encoding ActualEncoding = new UnicodeEncoding(bigEndian: false, byteOrderMark: false); - public override unsafe int GetByteCount(char* c, int i1) { throw new NotImplementedException(); } - public override int GetByteCount(char[] c, int i1, int i2) { throw new NotImplementedException(); } - public override int GetByteCount(string s) { throw new NotImplementedException(); } - public override int GetByteCount(char[] c) { throw new NotImplementedException(); } - public override byte[] GetBytes(char[] c) { throw new NotImplementedException(); } - public override byte[] GetBytes(char[] c, int i1, int i2) { throw new NotImplementedException(); } - public override int GetBytes(char[] c, int i1, int i2, byte[] b, int i3) { throw new NotImplementedException(); } - public override byte[] GetBytes(string s) { throw new NotImplementedException(); } - public override unsafe int GetBytes(char* c, int i1, byte* b, int i2) { throw new NotImplementedException(); } - public override int GetBytes(string s, int i1, int i2, byte[] b, int i3) { throw new NotImplementedException(); } - public override int GetCharCount(byte[] b) { throw new NotImplementedException(); } - public override int GetCharCount(byte[] b, int i1, int i2) { throw new NotImplementedException(); } - public override unsafe int GetCharCount(byte* b, int i1) { throw new NotImplementedException(); } - public override char[] GetChars(byte[] b, int i1, int i2) { throw new NotImplementedException(); } - public override int GetChars(byte[] b, int i1, int i2, char[] c, int i3) { throw new NotImplementedException(); } - public override unsafe int GetChars(byte* b, int i1, char* c, int i2) { throw new NotImplementedException(); } - public override char[] GetChars(byte[] b) { throw new NotImplementedException(); } - public override Decoder GetDecoder() { throw new NotImplementedException(); } - public override Encoder GetEncoder() { throw new NotImplementedException(); } - public override int GetHashCode() { throw new NotImplementedException(); } - public override int GetMaxByteCount(int i1) { throw new NotImplementedException(); } - public override int GetMaxCharCount(int i1) { throw new NotImplementedException(); } - public override byte[] GetPreamble() { throw new NotImplementedException(); } - public override string GetString(byte[] b) { throw new NotImplementedException(); } - public override string GetString(byte[] b, int i1, int i2) { throw new NotImplementedException(); } - public override bool IsAlwaysNormalized(NormalizationForm f) { throw new NotImplementedException(); } - } - } - - /// - /// To make it easier to specify -Encoding parameter, we add an ArgumentTransformationAttribute here. - /// When the input data is of type string and is valid to be converted to System.Text.Encoding, we do - /// the conversion and return the converted value. Otherwise, we just return the input data. - /// - internal sealed class ArgumentToEncodingTransformationAttribute : ArgumentTransformationAttribute - { - public override object Transform(EngineIntrinsics engineIntrinsics, object inputData) - { - string encodingName; - if (LanguagePrimitives.TryConvertTo(inputData, out encodingName)) - { - if (string.Equals(encodingName, EncodingConversion.Unknown, StringComparison.OrdinalIgnoreCase) || - string.Equals(encodingName, EncodingConversion.Byte, StringComparison.OrdinalIgnoreCase) || - string.Equals(encodingName, EncodingConversion.String, StringComparison.OrdinalIgnoreCase) || - string.Equals(encodingName, EncodingConversion.Unicode, StringComparison.OrdinalIgnoreCase) || - string.Equals(encodingName, EncodingConversion.BigEndianUnicode, StringComparison.OrdinalIgnoreCase) || - string.Equals(encodingName, EncodingConversion.Utf8, StringComparison.OrdinalIgnoreCase) || - string.Equals(encodingName, EncodingConversion.Utf8Bom, StringComparison.OrdinalIgnoreCase) || - string.Equals(encodingName, EncodingConversion.Utf8NoBom, StringComparison.OrdinalIgnoreCase) || - string.Equals(encodingName, EncodingConversion.Utf7, StringComparison.OrdinalIgnoreCase) || - string.Equals(encodingName, EncodingConversion.Utf32, StringComparison.OrdinalIgnoreCase) || - string.Equals(encodingName, EncodingConversion.Ascii, StringComparison.OrdinalIgnoreCase) || - string.Equals(encodingName, EncodingConversion.Default, StringComparison.OrdinalIgnoreCase) || - string.Equals(encodingName, EncodingConversion.OEM, StringComparison.OrdinalIgnoreCase)) - { - // the encodingName is guaranteed to be valid, so it is safe to pass null to method - // Convert(Cmdlet cmdlet, string encoding) as the value of 'cmdlet'. - return EncodingConversion.Convert(null, encodingName); - } - } - return inputData; - } - - } - - internal class EncodingArgumentCompleter : IArgumentCompleter - { - /// - /// - /// - public IEnumerable CompleteArgument( - string commandName, - string parameterName, - string wordToComplete, - System.Management.Automation.Language.CommandAst commandAst, - System.Collections.IDictionary fakeBoundParameters) - { - List encodings = new List(); - foreach(string encoding in EncodingConversion.TabCompletionResults) - { - if (string.IsNullOrEmpty(wordToComplete)) - { - encodings.Add(new CompletionResult(encoding, encoding, CompletionResultType.Text, encoding)); - } - else if ( encoding.IndexOf(wordToComplete, StringComparison.InvariantCultureIgnoreCase) == 0 ) - { - encodings.Add(new CompletionResult(encoding, encoding, CompletionResultType.Text, encoding)); - } - } - return encodings; - } - } } diff --git a/test/powershell/Language/Parser/RedirectionOperator.Tests.ps1 b/test/powershell/Language/Parser/RedirectionOperator.Tests.ps1 index 1e352999048..39669e0e6b0 100644 --- a/test/powershell/Language/Parser/RedirectionOperator.Tests.ps1 +++ b/test/powershell/Language/Parser/RedirectionOperator.Tests.ps1 @@ -10,7 +10,7 @@ Describe "Redirection operator now supports encoding changes" -Tags "CI" { } - # If out-file -encoding happens to have a default, be sure to + # If Out-File -Encoding happens to have a default, be sure to # save it away $SavedValue = $null $oldDefaultParameterValues = $psDefaultParameterValues.Clone() @@ -22,22 +22,22 @@ Describe "Redirection operator now supports encoding changes" -Tags "CI" { } BeforeEach { # start each test with a clean plate! - $psdefaultParameterValues.Remove("out-file:encoding") + $psdefaultParameterValues.Remove("Out-File:Encoding") } AfterEach { # end each test with a clean plate! - $psdefaultParameterValues.Remove("out-file:encoding") + $psdefaultParameterValues.Remove("Out-File:Encoding") } It "If encoding is unset, redirection should be UTF8 without bom" { $asciiString > TESTDRIVE:\file.txt - $bytes = get-content -encoding byte TESTDRIVE:\file.txt + $bytes = Get-Content -Encoding byte TESTDRIVE:\file.txt # create the expected - utf8 encoding without a bom - $encoding = [text.utf8encoding]::new($false) - $BOM = $encoding.GetPreamble() + $encoding = [Text.UTF8Encoding]::new($false) + # we know that there will be no preamble, so don't provide any bytes $TXT = $encoding.GetBytes($asciiString) $CR = $encoding.GetBytes($asciiCR) - $expectedBytes = .{ $BOM; $TXT; $CR } + $expectedBytes = .{ $TXT; $CR } $bytes.Count | should be $expectedBytes.count for($i = 0; $i -lt $bytes.count; $i++) { $bytes[$i] | Should be $expectedBytes[$i] @@ -45,7 +45,7 @@ Describe "Redirection operator now supports encoding changes" -Tags "CI" { } # $availableEncodings = "unknown","string","unicode","bigendianunicode","utf8","utf7", "utf32","ascii","default","oem" - $availableEncodings = (get-command out-file).Parameters["Encoding"].Attributes.ValidValues + $availableEncodings = (Get-Command Out-File).Parameters["Encoding"].Attributes.ValidValues foreach($encoding in $availableEncodings) { $skipTest = $false @@ -57,21 +57,21 @@ Describe "Redirection operator now supports encoding changes" -Tags "CI" { $skipTest = $true } - # some of the encodings accepted by out-file aren't real, - # and out-file has its own translation, so we'll + # some of the encodings accepted by Out-File aren't real, + # and Out-File has its own translation, so we'll # not do that logic here, but simply ignore those encodings # as they eventually are translated to "real" encoding - $enc = [system.text.encoding]::$encoding + $enc = [System.Text.Encoding]::$encoding if ( $enc ) { - $msg = "Overriding encoding for out-file is respected for $encoding" + $msg = "Overriding encoding for Out-File is respected for $encoding" $BOM = $enc.GetPreamble() $TXT = $enc.GetBytes($asciiString) $CR = $enc.GetBytes($asciiCR) $expectedBytes = .{ $BOM; $TXT; $CR } - $psdefaultparameterValues["out-file:encoding"] = "$encoding" + $psdefaultparameterValues["Out-File:Encoding"] = "$encoding" $asciiString > TESTDRIVE:/file.txt - $observedBytes = Get-Content -encoding Byte TESTDRIVE:/file.txt + $observedBytes = Get-Content -Encoding Byte TESTDRIVE:/file.txt # THE TEST It $msg -Skip:$skipTest { $observedBytes.Count | Should be $expectedBytes.Count From e29baacebfc7765d78a0ea0f83d36d31e4eb37d4 Mon Sep 17 00:00:00 2001 From: James Truher Date: Mon, 16 Oct 2017 16:11:14 -0700 Subject: [PATCH 5/8] Change Encoding to remove byte as a possible encoding value Add a new parameter to the filesystem provider cmdlets to indicate whether we want a byte stream Unify Send-MailMessage with other cmdlets --- .../commands/utility/CSVCommands.cs | 40 +------------ .../FormatAndOutput/format-hex/Format-Hex.cs | 17 +----- .../FormatAndOutput/out-file/Out-File.cs | 10 +--- .../utility/ImplicitRemotingCommands.cs | 22 +------- .../commands/utility/MatchString.cs | 22 +------- .../commands/utility/Send-MailMessage.cs | 50 ++--------------- .../commands/utility/XmlCommands.cs | 17 +----- .../namespaces/FileSystemProvider.cs | 31 ++++------ .../utils/EncodingUtils.cs | 56 +------------------ .../Parser/RedirectionOperator.Tests.ps1 | 4 +- .../Get-Content.Tests.ps1 | 2 +- ...mands.Cmdlets.NoNewlineParameter.Tests.ps1 | 6 +- .../engine/Module/NewModuleManifest.Tests.ps1 | 2 +- 13 files changed, 37 insertions(+), 242 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/CSVCommands.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/CSVCommands.cs index b5d456e4e6c..5562d4d4812 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/CSVCommands.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/CSVCommands.cs @@ -214,25 +214,7 @@ public SwitchParameter NoClobber [Parameter()] [ArgumentToEncodingTransformationAttribute()] [ArgumentCompleter(typeof(EncodingArgumentCompleter))] - public Encoding Encoding - { - get - { - return _encoding; - } - set - { - if ( value == EncodingConversion.byteEncoding ) - { - _encoding = EncodingConversion.byteEncoding.ActualEncoding; - } - else - { - _encoding = value; - } - } - } - private Encoding _encoding = ClrFacade.GetDefaultEncoding(); + public Encoding Encoding = ClrFacade.GetDefaultEncoding(); /// /// Property that sets append parameter. @@ -598,25 +580,7 @@ public SwitchParameter UseCulture [Parameter()] [ArgumentToEncodingTransformationAttribute()] [ArgumentCompleter(typeof(EncodingArgumentCompleter))] - public Encoding Encoding - { - get - { - return _encoding; - } - set - { - if ( value == EncodingConversion.byteEncoding ) - { - _encoding = EncodingConversion.byteEncoding.ActualEncoding; - } - else - { - _encoding = value; - } - } - } - private Encoding _encoding = ClrFacade.GetDefaultEncoding(); + public Encoding Encoding = ClrFacade.GetDefaultEncoding(); /// /// Avoid writing out duplicate warning messages when there are diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/format-hex/Format-Hex.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/format-hex/Format-Hex.cs index fe0a3174fa6..637e982217a 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/format-hex/Format-Hex.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/format-hex/Format-Hex.cs @@ -48,22 +48,7 @@ public sealed class FormatHex : PSCmdlet [Parameter(ParameterSetName = "ByInputObject")] [ArgumentToEncodingTransformationAttribute()] [ArgumentCompleter(typeof(EncodingArgumentCompleter))] - public Encoding Encoding - { - get { return _encoding; } - set - { - if ( value == EncodingConversion.byteEncoding ) - { - _encoding = EncodingConversion.byteEncoding.ActualEncoding; - } - else - { - _encoding = value; - } - } - } - private Encoding _encoding = ClrFacade.GetDefaultEncoding(); + public Encoding Encoding = ClrFacade.GetDefaultEncoding(); /// /// This parameter is no-op diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/out-file/Out-File.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/out-file/Out-File.cs index 79aab995105..e4d3b36e8ec 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/out-file/Out-File.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/out-file/Out-File.cs @@ -75,13 +75,7 @@ public string LiteralPath [Parameter(Position = 1)] [ArgumentToEncodingTransformationAttribute()] [ArgumentCompleter(typeof(EncodingArgumentCompleter))] - public Encoding Encoding - { - get { return _encoding; } - set { _encoding = value; } - } - - private Encoding _encoding = ClrFacade.GetDefaultEncoding(); + public Encoding Encoding { get; set; } = ClrFacade.GetDefaultEncoding(); /// /// Property that sets append parameter. @@ -187,7 +181,7 @@ private LineOutput InstantiateLineOutputInterface() PathUtils.MasterStreamOpen( this, FilePath, - _encoding, + Encoding, false, // defaultEncoding Append, Force, diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/ImplicitRemotingCommands.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/ImplicitRemotingCommands.cs index fd7d62c0cc0..2e65d4392d9 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/ImplicitRemotingCommands.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/ImplicitRemotingCommands.cs @@ -79,25 +79,7 @@ public SwitchParameter Force [Parameter] [ArgumentToEncodingTransformationAttribute()] [ArgumentCompleter(typeof(EncodingArgumentCompleter))] - public Encoding Encoding - { - get - { - return _encoding; - } - set - { - if ( value == EncodingConversion.byteEncoding ) - { - _encoding = EncodingConversion.byteEncoding.ActualEncoding; - } - else - { - _encoding = value; - } - } - } - private Encoding _encoding = ClrFacade.GetDefaultEncoding(); + public Encoding Encoding = ClrFacade.GetDefaultEncoding(); #endregion Parameters @@ -152,7 +134,7 @@ protected override void BeginProcessing() List generatedFiles = GenerateProxyModule( tempDirectory, Path.GetFileName(directory.FullName), - _encoding, + Encoding, _force, listOfCommandMetadata, alias2resolvedCommandName, diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/MatchString.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/MatchString.cs index 813e1f8c0c8..7ca7918dc15 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/MatchString.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/MatchString.cs @@ -1204,25 +1204,7 @@ public SwitchParameter AllMatches [Parameter] [ArgumentToEncodingTransformationAttribute()] [ArgumentCompleter(typeof(EncodingArgumentCompleter))] - public Encoding Encoding - { - get - { - return _textEncoding; - } - set - { - if ( value == EncodingConversion.byteEncoding ) - { - _textEncoding = EncodingConversion.byteEncoding.ActualEncoding; - } - else - { - _textEncoding = value; - } - } - } - private System.Text.Encoding _textEncoding = ClrFacade.GetDefaultEncoding(); + public Encoding Encoding = ClrFacade.GetDefaultEncoding(); /// /// The number of context lines to collect. If set to a @@ -1433,7 +1415,7 @@ private bool ProcessFile(string filename) using (FileStream fs = new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) { - using (StreamReader sr = new StreamReader(fs, _textEncoding)) + using (StreamReader sr = new StreamReader(fs, Encoding)) { String line; int lineNo = 0; diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Send-MailMessage.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Send-MailMessage.cs index a63ad2700e0..f257fb75245 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Send-MailMessage.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Send-MailMessage.cs @@ -89,20 +89,14 @@ public SwitchParameter BodyAsHtml /// /// Specifies the encoding used for the content of the body and also the subject. + /// This is set to ASCII to ensure there are no problems with any email server /// [Parameter()] [Alias("BE")] [ValidateNotNullOrEmpty] - [ArgumentToEncodingNameTransformationAttribute()] - public Encoding Encoding - { - get { return _encoding; } - set - { - _encoding = value; - } - } - private Encoding _encoding = new ASCIIEncoding(); + [ArgumentCompleter(typeof(EncodingArgumentCompleter))] + [ArgumentToEncodingTransformationAttribute()] + public Encoding Encoding { get; set; } = new ASCIIEncoding(); /// /// Specifies the address collection that contains the @@ -369,8 +363,8 @@ protected override _mMailMessage.Body = _body; //set the subject and body encoding - _mMailMessage.SubjectEncoding = _encoding; - _mMailMessage.BodyEncoding = _encoding; + _mMailMessage.SubjectEncoding = Encoding; + _mMailMessage.BodyEncoding = Encoding; // Set the format of the mail message body as HTML _mMailMessage.IsBodyHtml = _bodyashtml; @@ -490,37 +484,5 @@ protected override void EndProcessing() #endregion } - /// - /// To make it easier to specify -Encoding parameter, we add an ArgumentTransformationAttribute here. - /// When the input data is of type string and is valid to be converted to System.Text.Encoding, we do - /// the conversion and return the converted value. Otherwise, we just return the input data. - /// - internal sealed class ArgumentToEncodingNameTransformationAttribute : ArgumentTransformationAttribute - { - public override object Transform(EngineIntrinsics engineIntrinsics, object inputData) - { - string encodingName; - if (LanguagePrimitives.TryConvertTo(inputData, out encodingName)) - { - if (string.Equals(encodingName, EncodingConversion.Unknown, StringComparison.OrdinalIgnoreCase) || - string.Equals(encodingName, EncodingConversion.String, StringComparison.OrdinalIgnoreCase) || - string.Equals(encodingName, EncodingConversion.Unicode, StringComparison.OrdinalIgnoreCase) || - string.Equals(encodingName, EncodingConversion.BigEndianUnicode, StringComparison.OrdinalIgnoreCase) || - string.Equals(encodingName, EncodingConversion.Utf8, StringComparison.OrdinalIgnoreCase) || - string.Equals(encodingName, EncodingConversion.Utf7, StringComparison.OrdinalIgnoreCase) || - string.Equals(encodingName, EncodingConversion.Utf32, StringComparison.OrdinalIgnoreCase) || - string.Equals(encodingName, EncodingConversion.Ascii, StringComparison.OrdinalIgnoreCase) || - string.Equals(encodingName, EncodingConversion.Default, StringComparison.OrdinalIgnoreCase) || - string.Equals(encodingName, EncodingConversion.OEM, StringComparison.OrdinalIgnoreCase)) - { - // the encodingName is guaranteed to be valid, so it is safe to pass null to method - // Convert(Cmdlet cmdlet, string encoding) as the value of 'cmdlet'. - return EncodingConversion.Convert(null, encodingName); - } - } - return inputData; - } - } - #endregion } diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/XmlCommands.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/XmlCommands.cs index 3aa4bef7608..ba4fde033f9 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/XmlCommands.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/XmlCommands.cs @@ -110,22 +110,7 @@ public SwitchParameter NoClobber [Parameter] [ArgumentToEncodingTransformationAttribute()] [ArgumentCompleter(typeof(EncodingArgumentCompleter))] - public Encoding Encoding - { - get { return _encoding; } - set - { - if ( value == EncodingConversion.byteEncoding ) - { - _encoding = EncodingConversion.byteEncoding.ActualEncoding; - } - else - { - _encoding = value; - } - } - } - private Encoding _encoding = ClrFacade.GetDefaultEncoding(); + public Encoding Encoding = ClrFacade.GetDefaultEncoding(); #endregion Command Line Parameters diff --git a/src/System.Management.Automation/namespaces/FileSystemProvider.cs b/src/System.Management.Automation/namespaces/FileSystemProvider.cs index b193ae4c303..a24350580d3 100644 --- a/src/System.Management.Automation/namespaces/FileSystemProvider.cs +++ b/src/System.Management.Automation/namespaces/FileSystemProvider.cs @@ -6614,7 +6614,7 @@ public IContentReader GetContentReader(string path) delimiter = dynParams.Delimiter; // Get the stream type - usingByteEncoding = dynParams.UsingByteEncoding; + usingByteEncoding = dynParams.Byte; streamTypeSpecified = dynParams.WasStreamTypeSpecified; if (streamTypeSpecified) @@ -6765,7 +6765,7 @@ public IContentWriter GetContentWriter(string path) if (dynParams != null) { - usingByteEncoding = dynParams.UsingByteEncoding; + usingByteEncoding = dynParams.Byte; streamTypeSpecified = dynParams.WasStreamTypeSpecified; if (streamTypeSpecified) @@ -7590,20 +7590,19 @@ public Encoding Encoding } set { - if ( value == EncodingConversion.byteEncoding ) - { - _encoding = EncodingConversion.byteEncoding.ActualEncoding; - UsingByteEncoding = true; - } - else - { - _encoding = value; - } + _encoding = value; // If an encoding was explicitly set, be sure to capture that. WasStreamTypeSpecified = true; } } private Encoding _encoding = ClrFacade.GetDefaultEncoding(); + + /// + /// Return file contents as a byte stream or create file from a series of bytes + /// + [Parameter] + public SwitchParameter Byte { get; set; } + #if !UNIX /// /// A parameter to return a stream of an item. @@ -7612,12 +7611,6 @@ public Encoding Encoding public String Stream { get; set; } #endif - /// - /// Gets the Byte Encoding status of the StreamType parameter. Returns true - /// if the stream was opened with "Byte" encoding, false otherwise. - /// - public bool UsingByteEncoding { get; set; } - /// /// Gets the status of the StreamType parameter. Returns true /// if the stream was opened with a user-specified encoding, false otherwise. @@ -7680,13 +7673,13 @@ public string Delimiter get { return _delimiter; - } // get + } set { DelimiterSpecified = true; _delimiter = value; - } // set + } } private string _delimiter = "\n"; diff --git a/src/System.Management.Automation/utils/EncodingUtils.cs b/src/System.Management.Automation/utils/EncodingUtils.cs index bae66e7820e..2d2076163e8 100644 --- a/src/System.Management.Automation/utils/EncodingUtils.cs +++ b/src/System.Management.Automation/utils/EncodingUtils.cs @@ -26,27 +26,14 @@ internal static class EncodingConversion internal const string Utf32 = "utf32"; internal const string Default = "default"; internal const string OEM = "oem"; - internal const string Byte = "byte"; // This will return unicode, but 'byte' is special as it uses unicode encoding without a BOM - // We're using this custom ByteEncoding so the *-Content cmdlets can provide `byte` as an encoding, which has the - // behavior of [io.file]::ReadAllBytes(). When we changed the type of the parameter to Encoding - // and created the ArgumentTransformationAttribute to convert strings to encodings to be more usable, it means that - // we have to check to se - internal static readonly ByteEncoding byteEncoding = new ByteEncoding(); - // We will provide a partial list for tab completion we will convert to an encoding. - // If the user has an encoding object, we will cheerfully use it, but we can't use - // it as a validation set, as a user may have an specialty encoding which we have - // no idea about. - // We can't use Encoding.GetEncoding() as a validate set because we need to support - // unreal encodings like 'Byte' and 'OEM' internal static readonly string[] TabCompletionResults = { - Ascii, BigEndianUnicode, Byte, OEM, Unicode, Utf7, Utf8, Utf8Bom, Utf8NoBom, Utf32 + Ascii, BigEndianUnicode, OEM, Unicode, Utf7, Utf8, Utf8Bom, Utf8NoBom, Utf32 }; internal static Dictionary encodingMap = new Dictionary(StringComparer.OrdinalIgnoreCase) { { Ascii, System.Text.Encoding.ASCII }, { BigEndianUnicode, System.Text.Encoding.BigEndianUnicode }, - { Byte, byteEncoding }, { Default, ClrFacade.GetDefaultEncoding() }, { OEM, ClrFacade.GetOEMEncoding() }, { Unicode, System.Text.Encoding.Unicode }, @@ -94,45 +81,6 @@ internal static Encoding Convert(Cmdlet cmdlet, string encoding) return null; } - // We need a way to provide an encoding and also - // a way to notify that the user wants "byte" encoding which is just a stream of bytes - // Because we've changed the parameter type to Encoding, we need a way - // to continue to support the *-Content cmdlets use of -Encoding Byte, so we need - // to have the ArgumentTransformationAttribute actually return an encoding - // moreover, we need a way to notify the -Content cmdlets that the user requested `byte` - // so we can't just return the actual encoding we want, thus this artificial encoding - internal class ByteEncoding : System.Text.Encoding - { - // the encoding for this is not bigendian and not BOM - public Encoding ActualEncoding = new UnicodeEncoding(bigEndian: false, byteOrderMark: false); - // redirect all the calls to the ActualEncoding - public override unsafe int GetByteCount(char* chars, int count) { return ActualEncoding.GetByteCount(chars, count); } - public override int GetByteCount(char[] chars, int index, int count) { return ActualEncoding.GetByteCount(chars, index, count); } - public override int GetByteCount(string s) { return ActualEncoding.GetByteCount(s); } - public override int GetByteCount(char[] chars) { return ActualEncoding.GetByteCount(chars); } - public override byte[] GetBytes(char[] chars) { return ActualEncoding.GetBytes(chars); } - public override byte[] GetBytes(char[] chars, int index, int count) { return ActualEncoding.GetBytes(chars, index, count); } - public override int GetBytes(char[] chars, int charIndex, int charCount, byte[] bytes, int byteIndex) { return ActualEncoding.GetBytes(chars, charIndex, charCount, bytes, byteIndex); } - public override byte[] GetBytes(string s) { return ActualEncoding.GetBytes(s); } - public override unsafe int GetBytes(char* chars, int charCount, byte* bytes, int byteCount) { return ActualEncoding.GetBytes(chars, charCount, bytes, byteCount); } - public override int GetBytes(string s, int charIndex, int charCount, byte[] bytes, int byteIndex) { return ActualEncoding.GetBytes(s, charIndex, charCount, bytes, byteIndex); } - public override int GetCharCount(byte[] bytes) { return ActualEncoding.GetCharCount(bytes); } - public override int GetCharCount(byte[] bytes, int index, int count) { return ActualEncoding.GetCharCount(bytes, index, count); } - public override unsafe int GetCharCount(byte* bytes, int count) { return ActualEncoding.GetCharCount(bytes, count); } - public override char[] GetChars(byte[] bytes, int index, int count) { return ActualEncoding.GetChars(bytes, index, count); } - public override int GetChars(byte[] bytes, int byteIndex, int byteCount, char[] chars, int charIndex) { return ActualEncoding.GetChars(bytes, byteIndex, byteCount, chars, charIndex); } - public override unsafe int GetChars(byte* bytes, int byteCount, char* chars, int charCount) { return ActualEncoding.GetChars(bytes, byteCount, chars, charCount); } - public override char[] GetChars(byte[] bytes) { return ActualEncoding.GetChars(bytes); } - public override Decoder GetDecoder() { return ActualEncoding.GetDecoder(); } - public override Encoder GetEncoder() { return ActualEncoding.GetEncoder(); } - public override int GetHashCode() { return ActualEncoding.GetHashCode(); } - public override int GetMaxByteCount(int charCount) { return ActualEncoding.GetMaxByteCount(charCount); } - public override int GetMaxCharCount(int byteCount) { return ActualEncoding.GetMaxCharCount(byteCount); } - public override byte[] GetPreamble() { return ActualEncoding.GetPreamble(); } - public override string GetString(byte[] bytes) { return ActualEncoding.GetString(bytes); } - public override string GetString(byte[] bytes, int index, int count) { return ActualEncoding.GetString(bytes, index, count); } - public override bool IsAlwaysNormalized(NormalizationForm normalizationForm) { return ActualEncoding.IsAlwaysNormalized(normalizationForm); } - } } /// @@ -161,7 +109,7 @@ public override object Transform(EngineIntrinsics engineIntrinsics, object input internal class EncodingArgumentCompleter : IArgumentCompleter { /// - /// + /// This allows us to actually complete the string with tab complete. /// public IEnumerable CompleteArgument( string commandName, diff --git a/test/powershell/Language/Parser/RedirectionOperator.Tests.ps1 b/test/powershell/Language/Parser/RedirectionOperator.Tests.ps1 index 39669e0e6b0..82f8edf79e6 100644 --- a/test/powershell/Language/Parser/RedirectionOperator.Tests.ps1 +++ b/test/powershell/Language/Parser/RedirectionOperator.Tests.ps1 @@ -31,7 +31,7 @@ Describe "Redirection operator now supports encoding changes" -Tags "CI" { It "If encoding is unset, redirection should be UTF8 without bom" { $asciiString > TESTDRIVE:\file.txt - $bytes = Get-Content -Encoding byte TESTDRIVE:\file.txt + $bytes = Get-Content -Byte TESTDRIVE:\file.txt # create the expected - utf8 encoding without a bom $encoding = [Text.UTF8Encoding]::new($false) # we know that there will be no preamble, so don't provide any bytes @@ -71,7 +71,7 @@ Describe "Redirection operator now supports encoding changes" -Tags "CI" { $expectedBytes = .{ $BOM; $TXT; $CR } $psdefaultparameterValues["Out-File:Encoding"] = "$encoding" $asciiString > TESTDRIVE:/file.txt - $observedBytes = Get-Content -Encoding Byte TESTDRIVE:/file.txt + $observedBytes = Get-Content -Byte TESTDRIVE:/file.txt # THE TEST It $msg -Skip:$skipTest { $observedBytes.Count | Should be $expectedBytes.Count diff --git a/test/powershell/Modules/Microsoft.PowerShell.Management/Get-Content.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Management/Get-Content.Tests.ps1 index 1f68a3c8b86..09ae4f99d27 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Management/Get-Content.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Management/Get-Content.Tests.ps1 @@ -144,7 +144,7 @@ baz $result.Length | Should Be 3 $expected = "rld3${nl}Hell", '4,W', "rld4${nl}" for ($i = 0; $i -lt $result.Length ; $i++) { $result[$i] | Should BeExactly $expected[$i]} - $result=get-content -path $testPath -encoding:Byte -tail 10 + $result=get-content -path $testPath -Byte -tail 10 $result.Length | Should Be 10 if ($IsWindows) { $expected = 52, 44, 87, 111, 114, 108, 100, 52, 13, 10 diff --git a/test/powershell/Modules/Microsoft.PowerShell.Management/Pester.Commands.Cmdlets.NoNewlineParameter.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Management/Pester.Commands.Cmdlets.NoNewlineParameter.Tests.ps1 index fc3231b2445..5568755776d 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Management/Pester.Commands.Cmdlets.NoNewlineParameter.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Management/Pester.Commands.Cmdlets.NoNewlineParameter.Tests.ps1 @@ -6,19 +6,19 @@ Describe "Tests for -NoNewline parameter of Out-File, Add-Content and Set-Conten It "NoNewline parameter works on Out-File" { $temp = "${TESTDRIVE}/test1.txt" 1..5 | Out-File $temp -Encoding 'ASCII' -NoNewline - (Get-Content $temp -Encoding Byte).Count | Should Be 5 + (Get-Content $temp -Byte).Count | Should Be 5 } It "NoNewline parameter works on Set-Content" { $temp = "${TESTDRIVE}/test2.txt" Set-Content -Path $temp -Value 'a','b','c' -Encoding 'ASCII' -NoNewline - (Get-Content $temp -Encoding Byte).Count | Should Be 3 + (Get-Content $temp -Byte).Count | Should Be 3 } It "NoNewline parameter works on Add-Content" { $temp = "${TESTDRIVE}/test3.txt" $temp = New-TemporaryFile 1..9 | ForEach-Object {Add-Content -Path $temp -Value $_ -Encoding 'ASCII' -NoNewline} - (Get-Content $temp -Encoding Byte).Count | Should Be 9 + (Get-Content $temp -Byte).Count | Should Be 9 } } diff --git a/test/powershell/engine/Module/NewModuleManifest.Tests.ps1 b/test/powershell/engine/Module/NewModuleManifest.Tests.ps1 index 0604dc41553..411fecf889e 100644 --- a/test/powershell/engine/Module/NewModuleManifest.Tests.ps1 +++ b/test/powershell/engine/Module/NewModuleManifest.Tests.ps1 @@ -34,7 +34,7 @@ Describe "New-ModuleManifest tests" -tags "CI" { function TestNewModuleManifestEncoding { param ([byte[]]$expected) New-ModuleManifest -Path $testModulePath - (Get-Content -Encoding Byte -Path $testModulePath -TotalCount $expected.Length) -join ',' | Should Be ($expected -join ',') + (Get-Content -Byte -Path $testModulePath -TotalCount $expected.Length) -join ',' | Should Be ($expected -join ',') } It "Verify module manifest encoding" { From b59b0eed410a1303db84baa0ff668e50a6645a69 Mon Sep 17 00:00:00 2001 From: James Truher Date: Thu, 19 Oct 2017 12:16:32 -0700 Subject: [PATCH 6/8] Changed -Byte parameter to -AsByteStream Created a warning when both -AsByteStream and -Encoding are used together updated tests to check that the warning is emitted --- .../namespaces/FileSystemProvider.cs | 16 ++++-- .../resources/FileSystemProviderStrings.resx | 3 ++ .../Parser/RedirectionOperator.Tests.ps1 | 4 +- .../Get-Content.Tests.ps1 | 52 +++++++++--------- ...mands.Cmdlets.NoNewlineParameter.Tests.ps1 | 6 +-- .../Set-Content.Tests.ps1 | 54 +++++++++++-------- .../engine/Module/NewModuleManifest.Tests.ps1 | 2 +- 7 files changed, 81 insertions(+), 56 deletions(-) diff --git a/src/System.Management.Automation/namespaces/FileSystemProvider.cs b/src/System.Management.Automation/namespaces/FileSystemProvider.cs index a24350580d3..14001c9e32f 100644 --- a/src/System.Management.Automation/namespaces/FileSystemProvider.cs +++ b/src/System.Management.Automation/namespaces/FileSystemProvider.cs @@ -6614,9 +6614,14 @@ public IContentReader GetContentReader(string path) delimiter = dynParams.Delimiter; // Get the stream type - usingByteEncoding = dynParams.Byte; + usingByteEncoding = dynParams.AsByteStream; streamTypeSpecified = dynParams.WasStreamTypeSpecified; + if (usingByteEncoding && streamTypeSpecified) + { + WriteWarning(StringUtil.Format(FileSystemProviderStrings.EncodingNotUsed)); + } + if (streamTypeSpecified) { encoding = dynParams.Encoding; @@ -6765,9 +6770,14 @@ public IContentWriter GetContentWriter(string path) if (dynParams != null) { - usingByteEncoding = dynParams.Byte; + usingByteEncoding = dynParams.AsByteStream; streamTypeSpecified = dynParams.WasStreamTypeSpecified; + if (usingByteEncoding && streamTypeSpecified) + { + WriteWarning(StringUtil.Format(FileSystemProviderStrings.EncodingNotUsed)); + } + if (streamTypeSpecified) { encoding = dynParams.Encoding; @@ -7601,7 +7611,7 @@ public Encoding Encoding /// Return file contents as a byte stream or create file from a series of bytes /// [Parameter] - public SwitchParameter Byte { get; set; } + public SwitchParameter AsByteStream { get; set; } #if !UNIX /// diff --git a/src/System.Management.Automation/resources/FileSystemProviderStrings.resx b/src/System.Management.Automation/resources/FileSystemProviderStrings.resx index 696fefc7380..e9cf79a50d5 100644 --- a/src/System.Management.Automation/resources/FileSystemProviderStrings.resx +++ b/src/System.Management.Automation/resources/FileSystemProviderStrings.resx @@ -243,6 +243,9 @@ Cannot process path '{0}' because the target represents a reserved device name. + + Encoding not used when -AsByteStream specified. + Cannot proceed with byte encoding. When using byte encoding the content must be of type byte. diff --git a/test/powershell/Language/Parser/RedirectionOperator.Tests.ps1 b/test/powershell/Language/Parser/RedirectionOperator.Tests.ps1 index 82f8edf79e6..0b7d9a7f9e0 100644 --- a/test/powershell/Language/Parser/RedirectionOperator.Tests.ps1 +++ b/test/powershell/Language/Parser/RedirectionOperator.Tests.ps1 @@ -31,7 +31,7 @@ Describe "Redirection operator now supports encoding changes" -Tags "CI" { It "If encoding is unset, redirection should be UTF8 without bom" { $asciiString > TESTDRIVE:\file.txt - $bytes = Get-Content -Byte TESTDRIVE:\file.txt + $bytes = Get-Content -AsByteStream TESTDRIVE:\file.txt # create the expected - utf8 encoding without a bom $encoding = [Text.UTF8Encoding]::new($false) # we know that there will be no preamble, so don't provide any bytes @@ -71,7 +71,7 @@ Describe "Redirection operator now supports encoding changes" -Tags "CI" { $expectedBytes = .{ $BOM; $TXT; $CR } $psdefaultparameterValues["Out-File:Encoding"] = "$encoding" $asciiString > TESTDRIVE:/file.txt - $observedBytes = Get-Content -Byte TESTDRIVE:/file.txt + $observedBytes = Get-Content -AsByteStream TESTDRIVE:/file.txt # THE TEST It $msg -Skip:$skipTest { $observedBytes.Count | Should be $expectedBytes.Count diff --git a/test/powershell/Modules/Microsoft.PowerShell.Management/Get-Content.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Management/Get-Content.Tests.ps1 index 09ae4f99d27..af573035193 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Management/Get-Content.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Management/Get-Content.Tests.ps1 @@ -70,8 +70,8 @@ Describe "Get-Content" -Tags "CI" { Pop-Location } #[BugId(BugDatabase.WindowsOutOfBandReleases, 906022)] - It "should throw 'PSNotSupportedException' when you set-content to an unsupported provider" -Skip:($IsLinux -Or $IsMacOS) { - {get-content -path HKLM:\\software\\microsoft -ea stop} | Should Throw "IContentCmdletProvider interface is not implemented" + It "should throw 'PSNotSupportedException' when you Set-Content to an unsupported provider" -Skip:($IsLinux -Or $IsMacOS) { + {Get-Content -Path HKLM:\\software\\microsoft -EA stop} | Should Throw "IContentCmdletProvider interface is not implemented" } It 'Verifies -Tail reports a TailNotSupported error for unsupported providers' { {Get-Content -Path Variable:\PSHOME -Tail 1 -ErrorAction Stop} | ShouldBeErrorId 'TailNotSupported,Microsoft.PowerShell.Commands.GetContentCommand' @@ -110,41 +110,41 @@ baz $actual[0] | Should Be $expected } It "should Get-Content with a variety of -Tail and -ReadCount values" {#[DRT] - set-content -path $testPath "Hello,World","Hello2,World2","Hello3,World3","Hello4,World4" - $result=get-content -path $testPath -readcount:-1 -tail 5 + Set-Content -Path $testPath "Hello,World","Hello2,World2","Hello3,World3","Hello4,World4" + $result=Get-Content -Path $testPath -Readcount:-1 -Tail 5 $result.Length | Should Be 4 $expected = "Hello,World","Hello2,World2","Hello3,World3","Hello4,World4" for ($i = 0; $i -lt $result.Length ; $i++) { $result[$i] | Should BeExactly $expected[$i]} - $result=get-content -path $testPath -readcount 0 -tail 3 + $result=Get-Content -Path $testPath -Readcount 0 -Tail 3 $result.Length | Should Be 3 $expected = "Hello2,World2","Hello3,World3","Hello4,World4" for ($i = 0; $i -lt $result.Length ; $i++) { $result[$i] | Should BeExactly $expected[$i]} - $result=get-content -path $testPath -readcount 1 -tail 3 + $result=Get-Content -Path $testPath -Readcount 1 -Tail 3 $result.Length | Should Be 3 $expected = "Hello2,World2","Hello3,World3","Hello4,World4" for ($i = 0; $i -lt $result.Length ; $i++) { $result[$i] | Should BeExactly $expected[$i]} - $result=get-content -path $testPath -readcount 99999 -tail 3 + $result=Get-Content -Path $testPath -Readcount 99999 -Tail 3 $result.Length | Should Be 3 $expected = "Hello2,World2","Hello3,World3","Hello4,World4" for ($i = 0; $i -lt $result.Length ; $i++) { $result[$i] | Should BeExactly $expected[$i]} - $result=get-content -path $testPath -readcount 2 -tail 3 + $result=Get-Content -Path $testPath -Readcount 2 -Tail 3 $result.Length | Should Be 2 $expected = "Hello2,World2","Hello3,World3" $expected = $expected,"Hello4,World4" for ($i = 0; $i -lt $result.Length ; $i++) { $result[$i] | Should BeExactly $expected[$i]} - $result=get-content -path $testPath -readcount 2 -tail 2 + $result=Get-Content -Path $testPath -Readcount 2 -Tail 2 $result.Length | Should Be 2 $expected = "Hello3,World3","Hello4,World4" for ($i = 0; $i -lt $result.Length ; $i++) { $result[$i] | Should BeExactly $expected[$i]} - $result=get-content -path $testPath -delimiter "," -tail 2 + $result=Get-Content -Path $testPath -Delimiter "," -Tail 2 $result.Length | Should Be 2 $expected = "World3${nl}Hello4", "World4${nl}" for ($i = 0; $i -lt $result.Length ; $i++) { $result[$i] | Should BeExactly $expected[$i]} - $result=get-content -path $testPath -delimiter "o" -tail 3 + $result=Get-Content -Path $testPath -Delimiter "o" -Tail 3 $result.Length | Should Be 3 $expected = "rld3${nl}Hell", '4,W', "rld4${nl}" for ($i = 0; $i -lt $result.Length ; $i++) { $result[$i] | Should BeExactly $expected[$i]} - $result=get-content -path $testPath -Byte -tail 10 + $result=Get-Content -Path $testPath -AsByteStream -Tail 10 $result.Length | Should Be 10 if ($IsWindows) { $expected = 52, 44, 87, 111, 114, 108, 100, 52, 13, 10 @@ -154,9 +154,9 @@ baz for ($i = 0; $i -lt $result.Length ; $i++) { $result[$i] | Should BeExactly $expected[$i]} } #[BugId(BugDatabase.WindowsOutOfBandReleases, 905829)] - It "should get-content that matches the input string"{ - set-content $testPath "Hello,llllWorlld","Hello2,llllWorlld2" - $result=get-content $testPath -delimiter "ll" + It "should Get-Content that matches the input string"{ + Set-Content $testPath "Hello,llllWorlld","Hello2,llllWorlld2" + $result=Get-Content $testPath -Delimiter "ll" $result.Length | Should Be 9 $expected = 'He', 'o,', '', 'Wor', "d${nl}He", 'o2,', '', 'Wor', "d2${nl}" for ($i = 0; $i -lt $result.Length ; $i++) { $result[$i] | Should BeExactly $expected[$i]} @@ -169,7 +169,7 @@ baz Get-Content $testPath | Should BeExactly $testString } - It "Should support NTFS streams using -stream" -Skip:(!$IsWindows) { + It "Should support NTFS streams using -Stream" -Skip:(!$IsWindows) { Set-Content -Path $testPath -Stream hello -Value World Get-Content -Path $testPath | Should Be $testString Get-Content -Path $testPath -Stream hello | Should Be "World" @@ -189,12 +189,12 @@ baz } It "-Stream is not a valid parameter for on Linux/Mac" -Skip:($IsWindows) -TestCases @( - @{cmdlet="get-content"}, - @{cmdlet="set-content"}, - @{cmdlet="clear-content"}, - @{cmdlet="add-content"}, - @{cmdlet="get-item"}, - @{cmdlet="remove-item"} + @{cmdlet="Get-Content"}, + @{cmdlet="Set-Content"}, + @{cmdlet="Clear-Content"}, + @{cmdlet="Add-Content"}, + @{cmdlet="Get-Item"}, + @{cmdlet="Remove-Item"} ) { param($cmdlet) (Get-Command $cmdlet).Parameters["stream"] | Should BeNullOrEmpty @@ -236,7 +236,7 @@ baz $fileContent = $firstLine,$secondLine,$thirdLine,$fourthLine } BeforeEach{ - Set-content -Path $testPath $fileContent + Set-Content -Path $testPath $fileContent } It "Should return all lines when -Tail value is more than number of lines in the file"{ $result = Get-Content -Path $testPath -ReadCount -1 -Tail 5 -Encoding UTF7 @@ -282,6 +282,10 @@ baz $expected[1] = $thirdLine Compare-Object -ReferenceObject $expected -DifferenceObject $result | Should BeNullOrEmpty } + It "A warning should be emitted if both -AsByteStream and -Encoding are used together" { + [byte[]][char[]]"test" | Set-Content -Encoding Unicode -AsByteStream "${TESTDRIVE}\bfile.txt" -WarningVariable contentWarning *>$null + $contentWarning.Message | Should Match "-AsByteStream" + } } } @@ -294,7 +298,7 @@ Describe "Get-Content -Raw test" -Tags "CI" { @{character = "a`r`nb"; testname = "CRLF-separated files without trailing newline"; filename = "crlf-nt.txt"} ) { param ($character, $filename) - Set-Content -Encoding Ascii -NoNewline "$TestDrive\$filename" -value $character + Set-Content -Encoding Ascii -NoNewline "$TestDrive\$filename" -Value $character Get-Content -Raw "$TestDrive\$filename" | Should BeExactly $character } } diff --git a/test/powershell/Modules/Microsoft.PowerShell.Management/Pester.Commands.Cmdlets.NoNewlineParameter.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Management/Pester.Commands.Cmdlets.NoNewlineParameter.Tests.ps1 index 5568755776d..fab6e4ce6af 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Management/Pester.Commands.Cmdlets.NoNewlineParameter.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Management/Pester.Commands.Cmdlets.NoNewlineParameter.Tests.ps1 @@ -6,19 +6,19 @@ Describe "Tests for -NoNewline parameter of Out-File, Add-Content and Set-Conten It "NoNewline parameter works on Out-File" { $temp = "${TESTDRIVE}/test1.txt" 1..5 | Out-File $temp -Encoding 'ASCII' -NoNewline - (Get-Content $temp -Byte).Count | Should Be 5 + (Get-Content $temp -AsByteStream).Count | Should Be 5 } It "NoNewline parameter works on Set-Content" { $temp = "${TESTDRIVE}/test2.txt" Set-Content -Path $temp -Value 'a','b','c' -Encoding 'ASCII' -NoNewline - (Get-Content $temp -Byte).Count | Should Be 3 + (Get-Content $temp -AsByteStream).Count | Should Be 3 } It "NoNewline parameter works on Add-Content" { $temp = "${TESTDRIVE}/test3.txt" $temp = New-TemporaryFile 1..9 | ForEach-Object {Add-Content -Path $temp -Value $_ -Encoding 'ASCII' -NoNewline} - (Get-Content $temp -Byte).Count | Should Be 9 + (Get-Content $temp -AsByteStream).Count | Should Be 9 } } diff --git a/test/powershell/Modules/Microsoft.PowerShell.Management/Set-Content.Tests.ps1 b/test/powershell/Modules/Microsoft.PowerShell.Management/Set-Content.Tests.ps1 index 087680434d1..a63295fde4f 100644 --- a/test/powershell/Modules/Microsoft.PowerShell.Management/Set-Content.Tests.ps1 +++ b/test/powershell/Modules/Microsoft.PowerShell.Management/Set-Content.Tests.ps1 @@ -1,17 +1,25 @@ Describe "Set-Content cmdlet tests" -Tags "CI" { BeforeAll { $file1 = "file1.txt" - $filePath1 = join-path $testdrive $file1 + $filePath1 = Join-Path $testdrive $file1 # if the registry doesn't exist, don't run those tests - $skipRegistry = ! (test-path hklm:/) + $skipRegistry = ! (Test-Path hklm:/) } + + It "A warning should be emitted if both -AsByteStream and -Encoding are used together" { + $testfile = "${TESTDRIVE}\bfile.txt" + "test" | Set-Content $testfile + $result = Get-Content -AsByteStream -Encoding Unicode -Path $testfile -WarningVariable contentWarning *>$null + $contentWarning.Message | Should Match "-AsByteStream" + } + Context "Set-Content should create a file if it does not exist" { AfterEach { - Remove-Item -path $filePath1 -Force -ErrorAction SilentlyContinue + Remove-Item -Path $filePath1 -Force -ErrorAction SilentlyContinue } It "should create a file if it does not exist" { - set-content -path $filePath1 -value "$file1" - $result = Get-Content -path $filePath1 + Set-Content -Path $filePath1 -Value "$file1" + $result = Get-Content -Path $filePath1 $result| Should be "$file1" } } @@ -20,33 +28,33 @@ Describe "Set-Content cmdlet tests" -Tags "CI" { New-Item -Path $filePath1 -ItemType File -Force } It "should set-Content of testdrive\$file1" { - set-content -path $filePath1 -value "ExpectedContent" - $result = Get-Content -path $filePath1 + Set-Content -Path $filePath1 -Value "ExpectedContent" + $result = Get-Content -Path $filePath1 $result| Should be "ExpectedContent" } It "should return expected string from testdrive\$file1" { - $result = get-content -path $filePath1 + $result = Get-Content -Path $filePath1 $result | Should BeExactly "ExpectedContent" } It "should Set-Content to testdrive\dynamicfile.txt with dynamic parameters" { - set-content -path $testdrive\dynamicfile.txt -value "ExpectedContent" - $result = Get-Content -path $testdrive\dynamicfile.txt + Set-Content -Path $testdrive\dynamicfile.txt -Value "ExpectedContent" + $result = Get-Content -Path $testdrive\dynamicfile.txt $result| Should BeExactly "ExpectedContent" } It "should return expected string from testdrive\dynamicfile.txt" { - $result = get-content -path $testdrive\dynamicfile.txt + $result = Get-Content -Path $testdrive\dynamicfile.txt $result | Should BeExactly "ExpectedContent" } It "should remove existing content from testdrive\$file1 when the -Value is `$null" { - $AsItWas=get-content $filePath1 + $AsItWas=Get-Content $filePath1 $AsItWas |Should BeExactly "ExpectedContent" - set-content -path $filePath1 -value $null -ea stop - $AsItIs=get-content $filePath1 + Set-Content -Path $filePath1 -Value $null -EA stop + $AsItIs=Get-Content $filePath1 $AsItIs| Should Not Be $AsItWas } It "should throw 'ParameterArgumentValidationErrorNullNotAllowed' when -Path is `$null" { try { - set-content -path $null -value "ShouldNotWorkBecausePathIsNull" -ea stop + Set-Content -Path $null -Value "ShouldNotWorkBecausePathIsNull" -EA stop Throw "Previous statement unexpectedly succeeded..." } catch { @@ -55,16 +63,16 @@ Describe "Set-Content cmdlet tests" -Tags "CI" { } It "should throw 'ParameterArgumentValidationErrorNullNotAllowed' when -Path is `$()" { try { - set-content -path $() -value "ShouldNotWorkBecausePathIsInvalid" -ea stop + Set-Content -Path $() -Value "ShouldNotWorkBecausePathIsInvalid" -EA stop Throw "Previous statement unexpectedly succeeded..." } catch { $_.FullyQualifiedErrorId | Should Be "ParameterArgumentValidationErrorNullNotAllowed,Microsoft.PowerShell.Commands.SetContentCommand" } } - It "should throw 'PSNotSupportedException' when you set-content to an unsupported provider" -skip:$skipRegistry { + It "should throw 'PSNotSupportedException' when you Set-Content to an unsupported provider" -skip:$skipRegistry { try { - set-content -path HKLM:\\software\\microsoft -value "ShouldNotWorkBecausePathIsUnsupported" -ea stop + Set-Content -Path HKLM:\\software\\microsoft -Value "ShouldNotWorkBecausePathIsUnsupported" -EA stop Throw "Previous statement unexpectedly succeeded..." } catch { @@ -73,8 +81,8 @@ Describe "Set-Content cmdlet tests" -Tags "CI" { } #[BugId(BugDatabase.WindowsOutOfBandReleases, 9058182)] It "should be able to pass multiple [string]`$objects to Set-Content through the pipeline to output a dynamic Path file" { - "hello","world"|set-content $testdrive\dynamicfile2.txt - $result=get-content $testdrive\dynamicfile2.txt + "hello","world"|Set-Content $testdrive\dynamicfile2.txt + $result=Get-Content $testdrive\dynamicfile2.txt $result.length |Should be 2 $result[0] |Should be "hello" $result[1] |Should be "world" @@ -87,7 +95,7 @@ Describe "Set-Content should work for PSDrive with UNC path as root" -Tags @('CI $file1 = "file1.txt" #create a random folder $randomFolderName = "TestFolder_" + (Get-Random).ToString() - $randomFolderPath = join-path $testdrive $randomFolderName + $randomFolderPath = Join-Path $testdrive $randomFolderName $null = New-Item -Path $randomFolderPath -ItemType Directory -ErrorAction SilentlyContinue } # test is Pending due to https://github.com/PowerShell/PowerShell/issues/3883 @@ -97,8 +105,8 @@ Describe "Set-Content should work for PSDrive with UNC path as root" -Tags @('CI # create share net share testshare=$randomFolderPath /grant:everyone,FULL New-PSDrive -Name Foo -Root \\localhost\testshare -PSProvider FileSystem - set-content -path Foo:\$file1 -value "$file1" - $result = Get-Content -path Foo:\$file1 + Set-Content -Path Foo:\$file1 -Value "$file1" + $result = Get-Content -Path Foo:\$file1 $result| Should be "$file1" } finally diff --git a/test/powershell/engine/Module/NewModuleManifest.Tests.ps1 b/test/powershell/engine/Module/NewModuleManifest.Tests.ps1 index 411fecf889e..8508db33c1d 100644 --- a/test/powershell/engine/Module/NewModuleManifest.Tests.ps1 +++ b/test/powershell/engine/Module/NewModuleManifest.Tests.ps1 @@ -34,7 +34,7 @@ Describe "New-ModuleManifest tests" -tags "CI" { function TestNewModuleManifestEncoding { param ([byte[]]$expected) New-ModuleManifest -Path $testModulePath - (Get-Content -Byte -Path $testModulePath -TotalCount $expected.Length) -join ',' | Should Be ($expected -join ',') + (Get-Content -AsByteStream -Path $testModulePath -TotalCount $expected.Length) -join ',' | Should Be ($expected -join ',') } It "Verify module manifest encoding" { From 840343d0ee1feedc6967c3dc182e1952334d38ec Mon Sep 17 00:00:00 2001 From: James Truher Date: Fri, 20 Oct 2017 14:48:48 -0700 Subject: [PATCH 7/8] put ValidateNotNullOrEmpty attribute on Encoding parameters Also improve encoding completion to handle wildcards Also make sure that Encoding parameters are properties rather than fields. Remove unused OpenStreamReader which takes a string since it's no longer in use --- .../commands/utility/CSVCommands.cs | 6 ++++-- .../FormatAndOutput/format-hex/Format-Hex.cs | 3 ++- .../FormatAndOutput/out-file/Out-File.cs | 1 + .../utility/ImplicitRemotingCommands.cs | 3 ++- .../commands/utility/MatchString.cs | 3 ++- .../commands/utility/Send-MailMessage.cs | 2 +- .../commands/utility/XmlCommands.cs | 3 ++- .../namespaces/FileSystemProvider.cs | 7 ++++--- .../resources/FileSystemProviderStrings.resx | 2 +- .../utils/EncodingUtils.cs | 19 +++++++++++-------- .../utils/PathUtils.cs | 13 ------------- 11 files changed, 30 insertions(+), 32 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/CSVCommands.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/CSVCommands.cs index 5562d4d4812..df2c719e5a8 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/CSVCommands.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/CSVCommands.cs @@ -214,7 +214,8 @@ public SwitchParameter NoClobber [Parameter()] [ArgumentToEncodingTransformationAttribute()] [ArgumentCompleter(typeof(EncodingArgumentCompleter))] - public Encoding Encoding = ClrFacade.GetDefaultEncoding(); + [ValidateNotNullOrEmpty] + public Encoding Encoding { get; set; } = ClrFacade.GetDefaultEncoding(); /// /// Property that sets append parameter. @@ -580,7 +581,8 @@ public SwitchParameter UseCulture [Parameter()] [ArgumentToEncodingTransformationAttribute()] [ArgumentCompleter(typeof(EncodingArgumentCompleter))] - public Encoding Encoding = ClrFacade.GetDefaultEncoding(); + [ValidateNotNullOrEmpty] + public Encoding Encoding { get; set; } = ClrFacade.GetDefaultEncoding(); /// /// Avoid writing out duplicate warning messages when there are diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/format-hex/Format-Hex.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/format-hex/Format-Hex.cs index 637e982217a..c1e176aaece 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/format-hex/Format-Hex.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/format-hex/Format-Hex.cs @@ -48,7 +48,8 @@ public sealed class FormatHex : PSCmdlet [Parameter(ParameterSetName = "ByInputObject")] [ArgumentToEncodingTransformationAttribute()] [ArgumentCompleter(typeof(EncodingArgumentCompleter))] - public Encoding Encoding = ClrFacade.GetDefaultEncoding(); + [ValidateNotNullOrEmpty] + public Encoding Encoding { get; set; } = ClrFacade.GetDefaultEncoding(); /// /// This parameter is no-op diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/out-file/Out-File.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/out-file/Out-File.cs index e4d3b36e8ec..d8fb0cd6c12 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/out-file/Out-File.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/out-file/Out-File.cs @@ -75,6 +75,7 @@ public string LiteralPath [Parameter(Position = 1)] [ArgumentToEncodingTransformationAttribute()] [ArgumentCompleter(typeof(EncodingArgumentCompleter))] + [ValidateNotNullOrEmpty] public Encoding Encoding { get; set; } = ClrFacade.GetDefaultEncoding(); /// diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/ImplicitRemotingCommands.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/ImplicitRemotingCommands.cs index 2e65d4392d9..5d0e04c55ac 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/ImplicitRemotingCommands.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/ImplicitRemotingCommands.cs @@ -79,7 +79,8 @@ public SwitchParameter Force [Parameter] [ArgumentToEncodingTransformationAttribute()] [ArgumentCompleter(typeof(EncodingArgumentCompleter))] - public Encoding Encoding = ClrFacade.GetDefaultEncoding(); + [ValidateNotNullOrEmpty] + public Encoding Encoding { get; set; } = ClrFacade.GetDefaultEncoding(); #endregion Parameters diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/MatchString.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/MatchString.cs index 7ca7918dc15..011faf66d0d 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/MatchString.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/MatchString.cs @@ -1204,7 +1204,8 @@ public SwitchParameter AllMatches [Parameter] [ArgumentToEncodingTransformationAttribute()] [ArgumentCompleter(typeof(EncodingArgumentCompleter))] - public Encoding Encoding = ClrFacade.GetDefaultEncoding(); + [ValidateNotNullOrEmpty] + public Encoding Encoding { get; set; } = ClrFacade.GetDefaultEncoding(); /// /// The number of context lines to collect. If set to a diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Send-MailMessage.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Send-MailMessage.cs index f257fb75245..079d1e6af11 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Send-MailMessage.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Send-MailMessage.cs @@ -96,7 +96,7 @@ public SwitchParameter BodyAsHtml [ValidateNotNullOrEmpty] [ArgumentCompleter(typeof(EncodingArgumentCompleter))] [ArgumentToEncodingTransformationAttribute()] - public Encoding Encoding { get; set; } = new ASCIIEncoding(); + public Encoding Encoding { get; set; } = Encoding.ASCII; /// /// Specifies the address collection that contains the diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/XmlCommands.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/XmlCommands.cs index ba4fde033f9..6446bb46727 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/XmlCommands.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/XmlCommands.cs @@ -110,7 +110,8 @@ public SwitchParameter NoClobber [Parameter] [ArgumentToEncodingTransformationAttribute()] [ArgumentCompleter(typeof(EncodingArgumentCompleter))] - public Encoding Encoding = ClrFacade.GetDefaultEncoding(); + [ValidateNotNullOrEmpty] + public Encoding Encoding { get; set; } = ClrFacade.GetDefaultEncoding(); #endregion Command Line Parameters diff --git a/src/System.Management.Automation/namespaces/FileSystemProvider.cs b/src/System.Management.Automation/namespaces/FileSystemProvider.cs index 14001c9e32f..a90985532cd 100644 --- a/src/System.Management.Automation/namespaces/FileSystemProvider.cs +++ b/src/System.Management.Automation/namespaces/FileSystemProvider.cs @@ -6619,7 +6619,7 @@ public IContentReader GetContentReader(string path) if (usingByteEncoding && streamTypeSpecified) { - WriteWarning(StringUtil.Format(FileSystemProviderStrings.EncodingNotUsed)); + WriteWarning(FileSystemProviderStrings.EncodingNotUsed); } if (streamTypeSpecified) @@ -6775,7 +6775,7 @@ public IContentWriter GetContentWriter(string path) if (usingByteEncoding && streamTypeSpecified) { - WriteWarning(StringUtil.Format(FileSystemProviderStrings.EncodingNotUsed)); + WriteWarning(FileSystemProviderStrings.EncodingNotUsed); } if (streamTypeSpecified) @@ -7592,6 +7592,7 @@ public class FileSystemContentDynamicParametersBase [Parameter] [ArgumentToEncodingTransformationAttribute()] [ArgumentCompleter(typeof(EncodingArgumentCompleter))] + [ValidateNotNullOrEmpty] public Encoding Encoding { get @@ -7625,7 +7626,7 @@ public Encoding Encoding /// Gets the status of the StreamType parameter. Returns true /// if the stream was opened with a user-specified encoding, false otherwise. /// - public bool WasStreamTypeSpecified { get; set; } + public bool WasStreamTypeSpecified { get; private set; } } // class FileSystemContentDynamicParametersBase diff --git a/src/System.Management.Automation/resources/FileSystemProviderStrings.resx b/src/System.Management.Automation/resources/FileSystemProviderStrings.resx index e9cf79a50d5..36894977dca 100644 --- a/src/System.Management.Automation/resources/FileSystemProviderStrings.resx +++ b/src/System.Management.Automation/resources/FileSystemProviderStrings.resx @@ -244,7 +244,7 @@ Cannot process path '{0}' because the target represents a reserved device name. - Encoding not used when -AsByteStream specified. + Encoding not used when '-AsByteStream' specified. Cannot proceed with byte encoding. When using byte encoding the content must be of type byte. diff --git a/src/System.Management.Automation/utils/EncodingUtils.cs b/src/System.Management.Automation/utils/EncodingUtils.cs index 2d2076163e8..d362022f16a 100644 --- a/src/System.Management.Automation/utils/EncodingUtils.cs +++ b/src/System.Management.Automation/utils/EncodingUtils.cs @@ -92,14 +92,16 @@ internal sealed class ArgumentToEncodingTransformationAttribute : ArgumentTransf { public override object Transform(EngineIntrinsics engineIntrinsics, object inputData) { - string encodingName; - if (LanguagePrimitives.TryConvertTo(inputData, out encodingName)) + string encodingName = inputData as String; + Encoding foundEncoding; + + if ( inputData is Encoding ) { - Encoding foundEncoding; - if (EncodingConversion.encodingMap.TryGetValue(encodingName, out foundEncoding)) - { - return foundEncoding; - } + return inputData; + } + if (EncodingConversion.encodingMap.TryGetValue(encodingName, out foundEncoding)) + { + return foundEncoding; } return inputData; } @@ -119,13 +121,14 @@ public IEnumerable CompleteArgument( System.Collections.IDictionary fakeBoundParameters) { List encodings = new List(); + WildcardPattern wildcardPattern = WildcardPattern.Get(wordToComplete, WildcardOptions.IgnoreCase); foreach(string encoding in EncodingConversion.TabCompletionResults) { if (string.IsNullOrEmpty(wordToComplete)) { encodings.Add(new CompletionResult(encoding, encoding, CompletionResultType.Text, encoding)); } - else if (encoding.IndexOf(wordToComplete, StringComparison.InvariantCultureIgnoreCase) == 0) + else if (encoding.IndexOf(wordToComplete, StringComparison.InvariantCultureIgnoreCase) == 0 || wildcardPattern.IsMatch(encoding)) { encodings.Add(new CompletionResult(encoding, encoding, CompletionResultType.Text, encoding)); } diff --git a/src/System.Management.Automation/utils/PathUtils.cs b/src/System.Management.Automation/utils/PathUtils.cs index 5d6879ea750..62ffc54faea 100644 --- a/src/System.Management.Automation/utils/PathUtils.cs +++ b/src/System.Management.Automation/utils/PathUtils.cs @@ -194,19 +194,6 @@ internal static StreamReader OpenStreamReader(PSCmdlet command, string filePath, return new StreamReader(fileStream, encoding); } - internal static StreamReader OpenStreamReader(PSCmdlet command, string filePath, string encoding, bool isLiteralPath) - { - FileStream fileStream = OpenFileStream(filePath, command, isLiteralPath); - if (encoding == null) - { - return new StreamReader(fileStream); - } - else - { - return new StreamReader(fileStream, EncodingConversion.Convert(command, encoding)); - } - } - internal static FileStream OpenFileStream(string filePath, PSCmdlet command, bool isLiteralPath) { string resolvedPath = PathUtils.ResolveFilePath(filePath, command, isLiteralPath); From 007bae4cecb139d7394173bf4ccb0b77e3c021da Mon Sep 17 00:00:00 2001 From: James Truher Date: Mon, 23 Oct 2017 12:31:56 -0700 Subject: [PATCH 8/8] Change from IArgumentCompleter attribute to ArgumentCompletions attribute for encoding attribute. Simplify logic for converting string to encoding --- .../commands/utility/CSVCommands.cs | 24 +++++++++++-- .../FormatAndOutput/format-hex/Format-Hex.cs | 12 ++++++- .../FormatAndOutput/out-file/Out-File.cs | 12 ++++++- .../utility/ImplicitRemotingCommands.cs | 12 ++++++- .../commands/utility/MatchString.cs | 12 ++++++- .../commands/utility/Send-MailMessage.cs | 12 ++++++- .../commands/utility/XmlCommands.cs | 12 ++++++- .../namespaces/FileSystemProvider.cs | 12 ++++++- .../utils/EncodingUtils.cs | 35 +------------------ 9 files changed, 100 insertions(+), 43 deletions(-) diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/CSVCommands.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/CSVCommands.cs index df2c719e5a8..5e3e247a376 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/CSVCommands.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/CSVCommands.cs @@ -213,7 +213,17 @@ public SwitchParameter NoClobber /// [Parameter()] [ArgumentToEncodingTransformationAttribute()] - [ArgumentCompleter(typeof(EncodingArgumentCompleter))] + [ArgumentCompletions( + EncodingConversion.Ascii, + EncodingConversion.BigEndianUnicode, + EncodingConversion.OEM, + EncodingConversion.Unicode, + EncodingConversion.Utf7, + EncodingConversion.Utf8, + EncodingConversion.Utf8Bom, + EncodingConversion.Utf8NoBom, + EncodingConversion.Utf32 + )] [ValidateNotNullOrEmpty] public Encoding Encoding { get; set; } = ClrFacade.GetDefaultEncoding(); @@ -580,7 +590,17 @@ public SwitchParameter UseCulture /// [Parameter()] [ArgumentToEncodingTransformationAttribute()] - [ArgumentCompleter(typeof(EncodingArgumentCompleter))] + [ArgumentCompletions( + EncodingConversion.Ascii, + EncodingConversion.BigEndianUnicode, + EncodingConversion.OEM, + EncodingConversion.Unicode, + EncodingConversion.Utf7, + EncodingConversion.Utf8, + EncodingConversion.Utf8Bom, + EncodingConversion.Utf8NoBom, + EncodingConversion.Utf32 + )] [ValidateNotNullOrEmpty] public Encoding Encoding { get; set; } = ClrFacade.GetDefaultEncoding(); diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/format-hex/Format-Hex.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/format-hex/Format-Hex.cs index c1e176aaece..bc90a9536ba 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/format-hex/Format-Hex.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/format-hex/Format-Hex.cs @@ -47,7 +47,17 @@ public sealed class FormatHex : PSCmdlet /// [Parameter(ParameterSetName = "ByInputObject")] [ArgumentToEncodingTransformationAttribute()] - [ArgumentCompleter(typeof(EncodingArgumentCompleter))] + [ArgumentCompletions( + EncodingConversion.Ascii, + EncodingConversion.BigEndianUnicode, + EncodingConversion.OEM, + EncodingConversion.Unicode, + EncodingConversion.Utf7, + EncodingConversion.Utf8, + EncodingConversion.Utf8Bom, + EncodingConversion.Utf8NoBom, + EncodingConversion.Utf32 + )] [ValidateNotNullOrEmpty] public Encoding Encoding { get; set; } = ClrFacade.GetDefaultEncoding(); diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/out-file/Out-File.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/out-file/Out-File.cs index d8fb0cd6c12..d7c0d43c58c 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/out-file/Out-File.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/FormatAndOutput/out-file/Out-File.cs @@ -74,7 +74,17 @@ public string LiteralPath /// [Parameter(Position = 1)] [ArgumentToEncodingTransformationAttribute()] - [ArgumentCompleter(typeof(EncodingArgumentCompleter))] + [ArgumentCompletions( + EncodingConversion.Ascii, + EncodingConversion.BigEndianUnicode, + EncodingConversion.OEM, + EncodingConversion.Unicode, + EncodingConversion.Utf7, + EncodingConversion.Utf8, + EncodingConversion.Utf8Bom, + EncodingConversion.Utf8NoBom, + EncodingConversion.Utf32 + )] [ValidateNotNullOrEmpty] public Encoding Encoding { get; set; } = ClrFacade.GetDefaultEncoding(); diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/ImplicitRemotingCommands.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/ImplicitRemotingCommands.cs index 5d0e04c55ac..7cd0138844e 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/ImplicitRemotingCommands.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/ImplicitRemotingCommands.cs @@ -78,7 +78,17 @@ public SwitchParameter Force /// [Parameter] [ArgumentToEncodingTransformationAttribute()] - [ArgumentCompleter(typeof(EncodingArgumentCompleter))] + [ArgumentCompletions( + EncodingConversion.Ascii, + EncodingConversion.BigEndianUnicode, + EncodingConversion.OEM, + EncodingConversion.Unicode, + EncodingConversion.Utf7, + EncodingConversion.Utf8, + EncodingConversion.Utf8Bom, + EncodingConversion.Utf8NoBom, + EncodingConversion.Utf32 + )] [ValidateNotNullOrEmpty] public Encoding Encoding { get; set; } = ClrFacade.GetDefaultEncoding(); diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/MatchString.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/MatchString.cs index 011faf66d0d..e6bcadf02bf 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/MatchString.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/MatchString.cs @@ -1203,7 +1203,17 @@ public SwitchParameter AllMatches /// [Parameter] [ArgumentToEncodingTransformationAttribute()] - [ArgumentCompleter(typeof(EncodingArgumentCompleter))] + [ArgumentCompletions( + EncodingConversion.Ascii, + EncodingConversion.BigEndianUnicode, + EncodingConversion.OEM, + EncodingConversion.Unicode, + EncodingConversion.Utf7, + EncodingConversion.Utf8, + EncodingConversion.Utf8Bom, + EncodingConversion.Utf8NoBom, + EncodingConversion.Utf32 + )] [ValidateNotNullOrEmpty] public Encoding Encoding { get; set; } = ClrFacade.GetDefaultEncoding(); diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Send-MailMessage.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Send-MailMessage.cs index 079d1e6af11..58328c56b35 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Send-MailMessage.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/Send-MailMessage.cs @@ -94,7 +94,17 @@ public SwitchParameter BodyAsHtml [Parameter()] [Alias("BE")] [ValidateNotNullOrEmpty] - [ArgumentCompleter(typeof(EncodingArgumentCompleter))] + [ArgumentCompletions( + EncodingConversion.Ascii, + EncodingConversion.BigEndianUnicode, + EncodingConversion.OEM, + EncodingConversion.Unicode, + EncodingConversion.Utf7, + EncodingConversion.Utf8, + EncodingConversion.Utf8Bom, + EncodingConversion.Utf8NoBom, + EncodingConversion.Utf32 + )] [ArgumentToEncodingTransformationAttribute()] public Encoding Encoding { get; set; } = Encoding.ASCII; diff --git a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/XmlCommands.cs b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/XmlCommands.cs index 6446bb46727..4f87989f169 100644 --- a/src/Microsoft.PowerShell.Commands.Utility/commands/utility/XmlCommands.cs +++ b/src/Microsoft.PowerShell.Commands.Utility/commands/utility/XmlCommands.cs @@ -109,7 +109,17 @@ public SwitchParameter NoClobber /// [Parameter] [ArgumentToEncodingTransformationAttribute()] - [ArgumentCompleter(typeof(EncodingArgumentCompleter))] + [ArgumentCompletions( + EncodingConversion.Ascii, + EncodingConversion.BigEndianUnicode, + EncodingConversion.OEM, + EncodingConversion.Unicode, + EncodingConversion.Utf7, + EncodingConversion.Utf8, + EncodingConversion.Utf8Bom, + EncodingConversion.Utf8NoBom, + EncodingConversion.Utf32 + )] [ValidateNotNullOrEmpty] public Encoding Encoding { get; set; } = ClrFacade.GetDefaultEncoding(); diff --git a/src/System.Management.Automation/namespaces/FileSystemProvider.cs b/src/System.Management.Automation/namespaces/FileSystemProvider.cs index a90985532cd..d08125db5c2 100644 --- a/src/System.Management.Automation/namespaces/FileSystemProvider.cs +++ b/src/System.Management.Automation/namespaces/FileSystemProvider.cs @@ -7591,7 +7591,17 @@ public class FileSystemContentDynamicParametersBase /// [Parameter] [ArgumentToEncodingTransformationAttribute()] - [ArgumentCompleter(typeof(EncodingArgumentCompleter))] + [ArgumentCompletions( + EncodingConversion.Ascii, + EncodingConversion.BigEndianUnicode, + EncodingConversion.OEM, + EncodingConversion.Unicode, + EncodingConversion.Utf7, + EncodingConversion.Utf8, + EncodingConversion.Utf8Bom, + EncodingConversion.Utf8NoBom, + EncodingConversion.Utf32 + )] [ValidateNotNullOrEmpty] public Encoding Encoding { diff --git a/src/System.Management.Automation/utils/EncodingUtils.cs b/src/System.Management.Automation/utils/EncodingUtils.cs index d362022f16a..e98ef8968a1 100644 --- a/src/System.Management.Automation/utils/EncodingUtils.cs +++ b/src/System.Management.Automation/utils/EncodingUtils.cs @@ -94,12 +94,7 @@ public override object Transform(EngineIntrinsics engineIntrinsics, object input { string encodingName = inputData as String; Encoding foundEncoding; - - if ( inputData is Encoding ) - { - return inputData; - } - if (EncodingConversion.encodingMap.TryGetValue(encodingName, out foundEncoding)) + if (encodingName != null && EncodingConversion.encodingMap.TryGetValue(encodingName, out foundEncoding)) { return foundEncoding; } @@ -108,32 +103,4 @@ public override object Transform(EngineIntrinsics engineIntrinsics, object input } - internal class EncodingArgumentCompleter : IArgumentCompleter - { - /// - /// This allows us to actually complete the string with tab complete. - /// - public IEnumerable CompleteArgument( - string commandName, - string parameterName, - string wordToComplete, - System.Management.Automation.Language.CommandAst commandAst, - System.Collections.IDictionary fakeBoundParameters) - { - List encodings = new List(); - WildcardPattern wildcardPattern = WildcardPattern.Get(wordToComplete, WildcardOptions.IgnoreCase); - foreach(string encoding in EncodingConversion.TabCompletionResults) - { - if (string.IsNullOrEmpty(wordToComplete)) - { - encodings.Add(new CompletionResult(encoding, encoding, CompletionResultType.Text, encoding)); - } - else if (encoding.IndexOf(wordToComplete, StringComparison.InvariantCultureIgnoreCase) == 0 || wildcardPattern.IsMatch(encoding)) - { - encodings.Add(new CompletionResult(encoding, encoding, CompletionResultType.Text, encoding)); - } - } - return encodings; - } - } }