From 7cdc66cc6fd795c85315ca35216c489d6bed98b7 Mon Sep 17 00:00:00 2001 From: Mahmudul Hasan Date: Thu, 23 Jan 2025 00:24:19 +0600 Subject: [PATCH 1/6] Added configuration property to control trimming whitespaces in CSV parser --- src/ExcelDataReader/Core/CsvFormat/CsvAnalyzer.cs | 4 ++-- src/ExcelDataReader/Core/CsvFormat/CsvParser.cs | 10 +++++++--- src/ExcelDataReader/Core/CsvFormat/CsvWorkbook.cs | 6 ++++-- src/ExcelDataReader/Core/CsvFormat/CsvWorksheet.cs | 11 +++++++---- src/ExcelDataReader/ExcelCsvReader.cs | 4 ++-- src/ExcelDataReader/ExcelReaderConfiguration.cs | 5 +++++ src/ExcelDataReader/ExcelReaderFactory.cs | 2 +- 7 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/ExcelDataReader/Core/CsvFormat/CsvAnalyzer.cs b/src/ExcelDataReader/Core/CsvFormat/CsvAnalyzer.cs index fa245196..d36b686e 100644 --- a/src/ExcelDataReader/Core/CsvFormat/CsvAnalyzer.cs +++ b/src/ExcelDataReader/Core/CsvFormat/CsvAnalyzer.cs @@ -9,7 +9,7 @@ internal static class CsvAnalyzer /// Uses fallbackEncoding if there is no BOM. Throws DecoderFallbackException if there are invalid characters in the stream. /// Returns the separator whose average field count is closest to its max field count. /// - public static void Analyze(Stream stream, char[] separators, Encoding fallbackEncoding, int analyzeInitialCsvRows, char? quoteChar, out int fieldCount, out char autodetectSeparator, out Encoding autodetectEncoding, out int bomLength, out int rowCount) + public static void Analyze(Stream stream, char[] separators, Encoding fallbackEncoding, int analyzeInitialCsvRows, char? quoteChar, bool trimWhiteSpace, out int fieldCount, out char autodetectSeparator, out Encoding autodetectEncoding, out int bomLength, out int rowCount) { var bufferSize = 1024; var probeSize = 16; @@ -29,7 +29,7 @@ public static void Analyze(Stream stream, char[] separators, Encoding fallbackEn { separatorInfos[i] = new SeparatorInfo { - Buffer = new CsvParser(separators[i], autodetectEncoding, quoteChar) + Buffer = new CsvParser(separators[i], autodetectEncoding, quoteChar, trimWhiteSpace) }; } diff --git a/src/ExcelDataReader/Core/CsvFormat/CsvParser.cs b/src/ExcelDataReader/Core/CsvFormat/CsvParser.cs index a0fa2a17..35c5f571 100644 --- a/src/ExcelDataReader/Core/CsvFormat/CsvParser.cs +++ b/src/ExcelDataReader/Core/CsvFormat/CsvParser.cs @@ -7,7 +7,7 @@ namespace ExcelDataReader.Core.CsvFormat; /// internal sealed class CsvParser { - public CsvParser(char separator, Encoding encoding, char? quoteChar = null) + public CsvParser(char separator, Encoding encoding, char? quoteChar = null, bool trimWhiteSpace = true) { Separator = separator; QuoteChar = quoteChar; @@ -19,6 +19,8 @@ public CsvParser(char separator, Encoding encoding, char? quoteChar = null) CharBuffer = new char[bufferSize]; State = CsvState.PreValue; + + TrimWhiteSpace = trimWhiteSpace; } private enum CsvState @@ -52,6 +54,8 @@ private enum CsvState private List> RowsResult { get; set; } = []; + private bool TrimWhiteSpace { get; } + public void ParseBuffer(byte[] bytes, int offset, int count, out List> rows) { while (count > 0) @@ -122,7 +126,7 @@ private void ParseChar(char c, int bytesUsed) private bool ReadPreValue(char c, int bytesUsed) { - if (IsWhitespace(c)) + if (IsWhitespace(c) && TrimWhiteSpace) { return true; } @@ -172,7 +176,7 @@ private bool ReadValue(char c, int bytesUsed) } else { - if (IsWhitespace(c)) + if (IsWhitespace(c) && TrimWhiteSpace) { TrailingWhitespaceCount++; } diff --git a/src/ExcelDataReader/Core/CsvFormat/CsvWorkbook.cs b/src/ExcelDataReader/Core/CsvFormat/CsvWorkbook.cs index 3b0f0f05..ca5ab5ea 100644 --- a/src/ExcelDataReader/Core/CsvFormat/CsvWorkbook.cs +++ b/src/ExcelDataReader/Core/CsvFormat/CsvWorkbook.cs @@ -3,7 +3,7 @@ namespace ExcelDataReader.Core.CsvFormat; -internal sealed class CsvWorkbook(Stream stream, Encoding encoding, char[] autodetectSeparators, int analyzeInitialCsvRows, char? quoteChar = null) : IWorkbook +internal sealed class CsvWorkbook(Stream stream, Encoding encoding, char[] autodetectSeparators, int analyzeInitialCsvRows, char? quoteChar = null, bool trimWhiteSpace = true) : IWorkbook { public int ResultsCount => 1; @@ -19,9 +19,11 @@ internal sealed class CsvWorkbook(Stream stream, Encoding encoding, char[] autod public int AnalyzeInitialCsvRows { get; } = analyzeInitialCsvRows; + public bool TrimWhiteSpace { get; } = trimWhiteSpace; + public IEnumerable ReadWorksheets() { - yield return new CsvWorksheet(Stream, Encoding, AutodetectSeparators, AnalyzeInitialCsvRows, QuoteChar); + yield return new CsvWorksheet(Stream, Encoding, AutodetectSeparators, AnalyzeInitialCsvRows, QuoteChar, TrimWhiteSpace); } public NumberFormatString GetNumberFormatString(int index) diff --git a/src/ExcelDataReader/Core/CsvFormat/CsvWorksheet.cs b/src/ExcelDataReader/Core/CsvFormat/CsvWorksheet.cs index 3beedaac..3a03fd9a 100644 --- a/src/ExcelDataReader/Core/CsvFormat/CsvWorksheet.cs +++ b/src/ExcelDataReader/Core/CsvFormat/CsvWorksheet.cs @@ -4,15 +4,16 @@ namespace ExcelDataReader.Core.CsvFormat; internal sealed class CsvWorksheet : IWorksheet { - public CsvWorksheet(Stream stream, Encoding fallbackEncoding, char[] autodetectSeparators, int analyzeInitialCsvRows, char? quoteChar = null) + public CsvWorksheet(Stream stream, Encoding fallbackEncoding, char[] autodetectSeparators, int analyzeInitialCsvRows, char? quoteChar = null, bool trimWhiteSpace = true) { Stream = stream; QuoteChar = quoteChar; + TrimWhiteSpace = trimWhiteSpace; Stream.Seek(0, SeekOrigin.Begin); try { // Try as UTF-8 first, or use BOM if present - CsvAnalyzer.Analyze(Stream, autodetectSeparators, Encoding.UTF8, analyzeInitialCsvRows, quoteChar, out var fieldCount, out var separator, out var encoding, out var bomLength, out var rowCount); + CsvAnalyzer.Analyze(Stream, autodetectSeparators, Encoding.UTF8, analyzeInitialCsvRows, quoteChar, trimWhiteSpace, out var fieldCount, out var separator, out var encoding, out var bomLength, out var rowCount); FieldCount = fieldCount; AnalyzedRowCount = rowCount; AnalyzedPartial = analyzeInitialCsvRows > 0; @@ -25,7 +26,7 @@ public CsvWorksheet(Stream stream, Encoding fallbackEncoding, char[] autodetectS // If cannot parse as UTF-8, try fallback encoding Stream.Seek(0, SeekOrigin.Begin); - CsvAnalyzer.Analyze(Stream, autodetectSeparators, fallbackEncoding, analyzeInitialCsvRows, quoteChar, out var fieldCount, out var separator, out var encoding, out var bomLength, out var rowCount); + CsvAnalyzer.Analyze(Stream, autodetectSeparators, fallbackEncoding, analyzeInitialCsvRows, quoteChar, trimWhiteSpace, out var fieldCount, out var separator, out var encoding, out var bomLength, out var rowCount); FieldCount = fieldCount; AnalyzedRowCount = rowCount; AnalyzedPartial = analyzeInitialCsvRows > 0; @@ -74,6 +75,8 @@ public int RowCount public Column[] ColumnWidths => null; + public bool TrimWhiteSpace { get; } + private int BomLength { get; set; } private bool AnalyzedPartial { get; } @@ -85,7 +88,7 @@ public IEnumerable ReadRows() var bufferSize = 1024; var buffer = new byte[bufferSize]; var rowIndex = 0; - var csv = new CsvParser(Separator, Encoding, QuoteChar); + var csv = new CsvParser(Separator, Encoding, QuoteChar, TrimWhiteSpace); var skipBomBytes = BomLength; Stream.Seek(0, SeekOrigin.Begin); diff --git a/src/ExcelDataReader/ExcelCsvReader.cs b/src/ExcelDataReader/ExcelCsvReader.cs index 243f2c01..c048f8a5 100644 --- a/src/ExcelDataReader/ExcelCsvReader.cs +++ b/src/ExcelDataReader/ExcelCsvReader.cs @@ -5,9 +5,9 @@ namespace ExcelDataReader; internal sealed class ExcelCsvReader : ExcelDataReader { - public ExcelCsvReader(Stream stream, Encoding fallbackEncoding, char[] autodetectSeparators, int analyzeInitialCsvRows, char? quoteChar = null) + public ExcelCsvReader(Stream stream, Encoding fallbackEncoding, char[] autodetectSeparators, int analyzeInitialCsvRows, char? quoteChar = null, bool trimWhiteSpace = true) { - Workbook = new CsvWorkbook(stream, fallbackEncoding, autodetectSeparators, analyzeInitialCsvRows, quoteChar); + Workbook = new CsvWorkbook(stream, fallbackEncoding, autodetectSeparators, analyzeInitialCsvRows, quoteChar, trimWhiteSpace); // By default, the data reader is positioned on the first result. Reset(); diff --git a/src/ExcelDataReader/ExcelReaderConfiguration.cs b/src/ExcelDataReader/ExcelReaderConfiguration.cs index 26fbc691..78b4a560 100644 --- a/src/ExcelDataReader/ExcelReaderConfiguration.cs +++ b/src/ExcelDataReader/ExcelReaderConfiguration.cs @@ -28,6 +28,11 @@ public class ExcelReaderConfiguration /// public char? QuoteChar { get; set; } = '"'; + /// + /// Gets or sets a value indicating whether to trim white space values for CSV (Default 'true'). + /// + public bool TrimWhiteSpace { get; set; } = true; + /// /// Gets or sets a value indicating whether to leave the stream open after the IExcelDataReader object is disposed. Default: false. /// diff --git a/src/ExcelDataReader/ExcelReaderFactory.cs b/src/ExcelDataReader/ExcelReaderFactory.cs index aad18d40..a1051283 100644 --- a/src/ExcelDataReader/ExcelReaderFactory.cs +++ b/src/ExcelDataReader/ExcelReaderFactory.cs @@ -165,7 +165,7 @@ public static IExcelDataReader CreateCsvReader(Stream fileStream, ExcelReaderCon fileStream = new LeaveOpenStream(fileStream); } - return new ExcelCsvReader(fileStream, configuration.FallbackEncoding, configuration.AutodetectSeparators, configuration.AnalyzeInitialCsvRows, configuration.QuoteChar); + return new ExcelCsvReader(fileStream, configuration.FallbackEncoding, configuration.AutodetectSeparators, configuration.AnalyzeInitialCsvRows, configuration.QuoteChar, configuration.TrimWhiteSpace); } private static bool TryGetWorkbook(Stream fileStream, CompoundDocument document, out Stream stream) From 6340cdbcb914ce8502311f9d0d31e174f6ce3112 Mon Sep 17 00:00:00 2001 From: Mahmudul Hasan Date: Thu, 23 Jan 2025 00:35:25 +0600 Subject: [PATCH 2/6] Updated readme --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 86a60e24..9374a76e 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,10 @@ var reader = ExcelReaderFactory.CreateReader(stream, new ExcelReaderConfiguratio // (CSV only) AutodetectSeparators = new char[] { ',', ';', '\t', '|', '#' }, + // Gets or sets a value indicating whether to trim white space values for CSV (Default 'true'). + // (CSV only) + TrimWhiteSpace = true, + // Gets or sets a value indicating whether to leave the stream open after // the IExcelDataReader object is disposed. Default: false LeaveOpen = false, From 7dbd2dd3cbc29dcc27b5d9ead1adbdc2e2c62791 Mon Sep 17 00:00:00 2001 From: Mahmudul Hasan Date: Thu, 23 Jan 2025 01:10:03 +0600 Subject: [PATCH 3/6] Added test for parsing CSV without trimming white space --- .../ExcelCsvReaderTest.cs | 24 +++++++++++++++++++ src/TestData/csv/test_issue_566.csv | 2 ++ 2 files changed, 26 insertions(+) create mode 100644 src/TestData/csv/test_issue_566.csv diff --git a/src/ExcelDataReader.Tests/ExcelCsvReaderTest.cs b/src/ExcelDataReader.Tests/ExcelCsvReaderTest.cs index bc38eae0..23f51482 100644 --- a/src/ExcelDataReader.Tests/ExcelCsvReaderTest.cs +++ b/src/ExcelDataReader.Tests/ExcelCsvReaderTest.cs @@ -553,4 +553,28 @@ public void GitIssue580_ReadCsvWithCustomQuoteChar() Assert.That(row4, Is.EqualTo(new object[] { "Test3", "IJK", "ZZ", "10,143.27", "0.00" })); }); } + + [Test] + public void GitIssue566_ParseCsvWithoutTrimmingWhiteSpace() + { + var keepSpaceConfig = new ExcelReaderConfiguration + { + TrimWhiteSpace = false, + }; + + using var excelReader = ExcelReaderFactory.CreateCsvReader(Configuration.GetTestWorkbook("csv\\test_issue_566.csv"), keepSpaceConfig); + + var ds = excelReader.AsDataSet(); + Assert.That(ds.Tables[0].Rows[0][0], Is.EqualTo("c1")); + Assert.That(ds.Tables[0].Rows[0][1], Is.EqualTo("c2 ")); + Assert.That(ds.Tables[0].Rows[0][2], Is.EqualTo("c3")); + Assert.That(ds.Tables[0].Rows[0][3], Is.EqualTo(" c4")); + Assert.That(ds.Tables[0].Rows[0][4], Is.EqualTo("c5")); + + Assert.That(ds.Tables[0].Rows[1][0], Is.EqualTo("11 ")); + Assert.That(ds.Tables[0].Rows[1][1], Is.EqualTo("12")); + Assert.That(ds.Tables[0].Rows[1][2], Is.EqualTo("\t")); + Assert.That(ds.Tables[0].Rows[1][3], Is.EqualTo(" ")); + Assert.That(ds.Tables[0].Rows[1][4], Is.EqualTo("15")); + } } \ No newline at end of file diff --git a/src/TestData/csv/test_issue_566.csv b/src/TestData/csv/test_issue_566.csv new file mode 100644 index 00000000..ad23c296 --- /dev/null +++ b/src/TestData/csv/test_issue_566.csv @@ -0,0 +1,2 @@ +c1,c2 ,c3, c4,c5 +11 ,12, , ,15 \ No newline at end of file From 5f785ea7e6a3f20dcb811fa00f5bbe9d63df319a Mon Sep 17 00:00:00 2001 From: Mahmudul Hasan Date: Thu, 23 Jan 2025 01:48:55 +0600 Subject: [PATCH 4/6] Fixed CA2263 violation --- src/ExcelDataReader/Core/OfficeCrypto/AgileEncryption.cs | 6 +++++- .../Core/OpenXmlFormat/XmlFormat/XmlStylesReader.cs | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/ExcelDataReader/Core/OfficeCrypto/AgileEncryption.cs b/src/ExcelDataReader/Core/OfficeCrypto/AgileEncryption.cs index 11b59446..a8c4d3a9 100644 --- a/src/ExcelDataReader/Core/OfficeCrypto/AgileEncryption.cs +++ b/src/ExcelDataReader/Core/OfficeCrypto/AgileEncryption.cs @@ -155,7 +155,11 @@ private static byte[] HashPassword(string password, byte[] saltValue, HashAlgori private static HashIdentifier ParseHash(string value) { - return (HashIdentifier)Enum.Parse(typeof(HashIdentifier), value); + #if NET8_0 + return Enum.Parse(value); + #else + return (HashIdentifier)Enum.Parse(typeof(HashIdentifier), value); + #endif } private static CipherIdentifier ParseCipher(string value/*, int blockBits*/) diff --git a/src/ExcelDataReader/Core/OpenXmlFormat/XmlFormat/XmlStylesReader.cs b/src/ExcelDataReader/Core/OpenXmlFormat/XmlFormat/XmlStylesReader.cs index ac1c6c9c..473ea919 100644 --- a/src/ExcelDataReader/Core/OpenXmlFormat/XmlFormat/XmlStylesReader.cs +++ b/src/ExcelDataReader/Core/OpenXmlFormat/XmlFormat/XmlStylesReader.cs @@ -132,7 +132,11 @@ static void ReadAlignment(XmlReader reader, string nsSpreadsheetMl, out int inde int.TryParse(reader.GetAttribute(AIndent), NumberStyles.Integer, CultureInfo.InvariantCulture, out indentLevel); try { + #if NET8_0 + horizontalAlignment = Enum.Parse(reader.GetAttribute(AHorizontal), true); + #else horizontalAlignment = (HorizontalAlignment)Enum.Parse(typeof(HorizontalAlignment), reader.GetAttribute(AHorizontal), true); + #endif } catch (ArgumentException) { From 065c0e631d84f3219282d6a084e864a6d657fa58 Mon Sep 17 00:00:00 2001 From: Mahmudul Hasan Date: Thu, 23 Jan 2025 01:55:26 +0600 Subject: [PATCH 5/6] Revert "Fixed CA2263 violation" This reverts commit 5f785ea7e6a3f20dcb811fa00f5bbe9d63df319a. --- src/ExcelDataReader/Core/OfficeCrypto/AgileEncryption.cs | 6 +----- .../Core/OpenXmlFormat/XmlFormat/XmlStylesReader.cs | 4 ---- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/src/ExcelDataReader/Core/OfficeCrypto/AgileEncryption.cs b/src/ExcelDataReader/Core/OfficeCrypto/AgileEncryption.cs index a8c4d3a9..11b59446 100644 --- a/src/ExcelDataReader/Core/OfficeCrypto/AgileEncryption.cs +++ b/src/ExcelDataReader/Core/OfficeCrypto/AgileEncryption.cs @@ -155,11 +155,7 @@ private static byte[] HashPassword(string password, byte[] saltValue, HashAlgori private static HashIdentifier ParseHash(string value) { - #if NET8_0 - return Enum.Parse(value); - #else - return (HashIdentifier)Enum.Parse(typeof(HashIdentifier), value); - #endif + return (HashIdentifier)Enum.Parse(typeof(HashIdentifier), value); } private static CipherIdentifier ParseCipher(string value/*, int blockBits*/) diff --git a/src/ExcelDataReader/Core/OpenXmlFormat/XmlFormat/XmlStylesReader.cs b/src/ExcelDataReader/Core/OpenXmlFormat/XmlFormat/XmlStylesReader.cs index 473ea919..ac1c6c9c 100644 --- a/src/ExcelDataReader/Core/OpenXmlFormat/XmlFormat/XmlStylesReader.cs +++ b/src/ExcelDataReader/Core/OpenXmlFormat/XmlFormat/XmlStylesReader.cs @@ -132,11 +132,7 @@ static void ReadAlignment(XmlReader reader, string nsSpreadsheetMl, out int inde int.TryParse(reader.GetAttribute(AIndent), NumberStyles.Integer, CultureInfo.InvariantCulture, out indentLevel); try { - #if NET8_0 - horizontalAlignment = Enum.Parse(reader.GetAttribute(AHorizontal), true); - #else horizontalAlignment = (HorizontalAlignment)Enum.Parse(typeof(HorizontalAlignment), reader.GetAttribute(AHorizontal), true); - #endif } catch (ArgumentException) { From cf796c3a5ec6184ac0571b4f521a16651599d10d Mon Sep 17 00:00:00 2001 From: Mahmudul Hasan Date: Thu, 23 Jan 2025 01:58:31 +0600 Subject: [PATCH 6/6] Fixed violation of CA2263 --- src/ExcelDataReader/Core/OfficeCrypto/AgileEncryption.cs | 2 ++ .../Core/OpenXmlFormat/XmlFormat/XmlStylesReader.cs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/ExcelDataReader/Core/OfficeCrypto/AgileEncryption.cs b/src/ExcelDataReader/Core/OfficeCrypto/AgileEncryption.cs index 11b59446..c954047a 100644 --- a/src/ExcelDataReader/Core/OfficeCrypto/AgileEncryption.cs +++ b/src/ExcelDataReader/Core/OfficeCrypto/AgileEncryption.cs @@ -155,7 +155,9 @@ private static byte[] HashPassword(string password, byte[] saltValue, HashAlgori private static HashIdentifier ParseHash(string value) { +#pragma warning disable CA2263 return (HashIdentifier)Enum.Parse(typeof(HashIdentifier), value); +#pragma warning restore CA2263 } private static CipherIdentifier ParseCipher(string value/*, int blockBits*/) diff --git a/src/ExcelDataReader/Core/OpenXmlFormat/XmlFormat/XmlStylesReader.cs b/src/ExcelDataReader/Core/OpenXmlFormat/XmlFormat/XmlStylesReader.cs index ac1c6c9c..18fc6dc3 100644 --- a/src/ExcelDataReader/Core/OpenXmlFormat/XmlFormat/XmlStylesReader.cs +++ b/src/ExcelDataReader/Core/OpenXmlFormat/XmlFormat/XmlStylesReader.cs @@ -132,7 +132,9 @@ static void ReadAlignment(XmlReader reader, string nsSpreadsheetMl, out int inde int.TryParse(reader.GetAttribute(AIndent), NumberStyles.Integer, CultureInfo.InvariantCulture, out indentLevel); try { +#pragma warning disable CA2263 horizontalAlignment = (HorizontalAlignment)Enum.Parse(typeof(HorizontalAlignment), reader.GetAttribute(AHorizontal), true); +#pragma warning restore CA2263 } catch (ArgumentException) {