diff --git a/src/ExcelDataReader.Tests/ExcelCsvReaderTest.cs b/src/ExcelDataReader.Tests/ExcelCsvReaderTest.cs index b8e0b10b..bc38eae0 100644 --- a/src/ExcelDataReader.Tests/ExcelCsvReaderTest.cs +++ b/src/ExcelDataReader.Tests/ExcelCsvReaderTest.cs @@ -3,7 +3,7 @@ namespace ExcelDataReader.Tests; /// -/// Most CSV test data came from csv-spectrum: https://github.com/maxogden/csv-spectrum. +/// Most CSV test data came from csv-spectrum: https://github.com/maxogden/csv-spectrum. /// public class ExcelCsvReaderTest { @@ -187,7 +187,7 @@ public void CsvWrongEncoding() }; using (ExcelReaderFactory.CreateCsvReader(Configuration.GetTestWorkbook("csv\\cp1252.csv"), configuration)) - { + { } }); } @@ -354,20 +354,18 @@ public void CsvRowCountAnalyzeRowsThrows() public void GitIssue578() { var stream = Configuration.GetTestWorkbook(@"csv\Test_git_issue578.csv"); - using (IExcelDataReader excelReader = ExcelReaderFactory.CreateCsvReader(stream, new ExcelReaderConfiguration())) - { - excelReader.Read(); - var values = new object[excelReader.FieldCount]; - excelReader.GetValues(values); - var values2 = new object[excelReader.FieldCount + 1]; - excelReader.GetValues(values2); - var values3 = new object[excelReader.FieldCount - 1]; - excelReader.GetValues(values3); - - Assert.That(values, Is.EqualTo(new object[] { "1", "2", "3", "4", "5" })); - Assert.That(values2, Is.EqualTo(new object[] { "1", "2", "3", "4", "5", null })); - Assert.That(values3, Is.EqualTo(new object[] { "1", "2", "3", "4" })); - } + using IExcelDataReader excelReader = ExcelReaderFactory.CreateCsvReader(stream, new ExcelReaderConfiguration()); + excelReader.Read(); + var values = new object[excelReader.FieldCount]; + excelReader.GetValues(values); + var values2 = new object[excelReader.FieldCount + 1]; + excelReader.GetValues(values2); + var values3 = new object[excelReader.FieldCount - 1]; + excelReader.GetValues(values3); + + Assert.That(values, Is.EqualTo(new object[] { "1", "2", "3", "4", "5" })); + Assert.That(values2, Is.EqualTo(new object[] { "1", "2", "3", "4", "5", null })); + Assert.That(values3, Is.EqualTo(new object[] { "1", "2", "3", "4" })); } [Test] @@ -377,7 +375,6 @@ public void GitIssue500() Item_Number,Pmt_Amount,Type,Voided,Note 200212812,$462.76,Check,06/06/2018,Hash#hash - """; using var reader = ExcelReaderFactory.CreateCsvReader(new MemoryStream(Encoding.UTF8.GetBytes(data))); @@ -400,11 +397,10 @@ public void GitIssue500_QuotedValueWithNewLine() { const string data = """ Item_Number,Pmt_Amount,"Type - + 2",Voided,Note 200212812,$462.76,Check,06/06/2018,Hash#hash - """; using var reader = ExcelReaderFactory.CreateCsvReader(new MemoryStream(Encoding.UTF8.GetBytes(data))); @@ -417,15 +413,15 @@ public void GitIssue500_QuotedValueWithNewLine() Assert.Multiple(() => { - Assert.That(row1, Is.EqualTo(new object[] + Assert.That(row1, Is.EqualTo(new object[] { - "Item_Number", + "Item_Number", "Pmt_Amount", """ Type - + 2 - """, - "Voided", + """, + "Voided", "Note", })); Assert.That(row2, Is.EqualTo(new object[] { "200212812", "$462.76", "Check", "06/06/2018", "Hash#hash" })); @@ -479,4 +475,82 @@ public void GitIssue642_ActiveSheet() Assert.That(reader.ActiveSheet, Is.EqualTo(0)); Assert.That(dataSet.Tables.Count, Is.EqualTo(1)); } -} + + [Test] + public void GitIssue580_ReadCsvWithoutQuoteChar() + { +#pragma warning disable SA1027 // TabsMustNotBeUsed + const string data = """ + Name Currency Ty"pe Cost Cost + Test1 A"BC XX 10,143.27 "0.00 + Test2 EFG YY 10,143.27 0.00 + Test3 IJK ZZ" 10,143.27 0.00 + + """; +#pragma warning restore SA1027 // TabsMustNotBeUsed + + using var reader = ExcelReaderFactory.CreateCsvReader(new MemoryStream(Encoding.UTF8.GetBytes(data)), new() + { + QuoteChar = null + }); + reader.Read(); + object[] row1 = new object[reader.FieldCount]; + reader.GetValues(row1); + reader.Read(); + object[] row2 = new object[reader.FieldCount]; + reader.GetValues(row2); + reader.Read(); + object[] row3 = new object[reader.FieldCount]; + reader.GetValues(row3); + reader.Read(); + object[] row4 = new object[reader.FieldCount]; + reader.GetValues(row4); + + Assert.Multiple(() => + { + Assert.That(row1, Is.EqualTo(new object[] { "Name", "Currency", "Ty\"pe", "Cost", "Cost" })); + Assert.That(row2, Is.EqualTo(new object[] { "Test1", "A\"BC", "XX", "10,143.27", "\"0.00" })); + Assert.That(row3, Is.EqualTo(new object[] { "Test2", "EFG", "YY", "10,143.27", "0.00" })); + Assert.That(row4, Is.EqualTo(new object[] { "Test3", "IJK", "ZZ\"", "10,143.27", "0.00" })); + }); + } + + [Test] + public void GitIssue580_ReadCsvWithCustomQuoteChar() + { +#pragma warning disable SA1027 // TabsMustNotBeUsed + const string data = """ + Name Currency Type Cost 'Cost per 1,000 Items' + Test1 ABC XX '10,143.27' 0.00 + Test2 EFG YY '10,143.27' 0.00 + Test3 IJK ZZ '10,143.27' 0.00 + + """; +#pragma warning restore SA1027 // TabsMustNotBeUsed + + using var reader = ExcelReaderFactory.CreateCsvReader(new MemoryStream(Encoding.UTF8.GetBytes(data)), new() + { + QuoteChar = '\'' + }); + reader.Read(); + object[] row1 = new object[reader.FieldCount]; + reader.GetValues(row1); + reader.Read(); + object[] row2 = new object[reader.FieldCount]; + reader.GetValues(row2); + reader.Read(); + object[] row3 = new object[reader.FieldCount]; + reader.GetValues(row3); + reader.Read(); + object[] row4 = new object[reader.FieldCount]; + reader.GetValues(row4); + + Assert.Multiple(() => + { + Assert.That(row1, Is.EqualTo(new object[] { "Name", "Currency", "Type", "Cost", "Cost per 1,000 Items" })); + Assert.That(row2, Is.EqualTo(new object[] { "Test1", "ABC", "XX", "10,143.27", "0.00" })); + Assert.That(row3, Is.EqualTo(new object[] { "Test2", "EFG", "YY", "10,143.27", "0.00" })); + Assert.That(row4, Is.EqualTo(new object[] { "Test3", "IJK", "ZZ", "10,143.27", "0.00" })); + }); + } +} \ No newline at end of file diff --git a/src/ExcelDataReader/Core/CsvFormat/CsvAnalyzer.cs b/src/ExcelDataReader/Core/CsvFormat/CsvAnalyzer.cs index 36ff0a89..fa245196 100644 --- a/src/ExcelDataReader/Core/CsvFormat/CsvAnalyzer.cs +++ b/src/ExcelDataReader/Core/CsvFormat/CsvAnalyzer.cs @@ -5,11 +5,11 @@ namespace ExcelDataReader.Core.CsvFormat; internal static class CsvAnalyzer { /// - /// Reads completely through a CSV stream to determine encoding, separator, field count and row count. + /// Reads completely through a CSV stream to determine encoding, separator, field count and row count. /// Uses fallbackEncoding if there is no BOM. Throws DecoderFallbackException if there are invalid characters in the stream. /// Returns the separator whose average field count is closest to its max field count. /// - public static void Analyze(Stream stream, char[] separators, Encoding fallbackEncoding, int analyzeInitialCsvRows, out int fieldCount, out char autodetectSeparator, out Encoding autodetectEncoding, out int bomLength, out int rowCount) + public static void Analyze(Stream stream, char[] separators, Encoding fallbackEncoding, int analyzeInitialCsvRows, char? quoteChar, out int fieldCount, out char autodetectSeparator, out Encoding autodetectEncoding, out int bomLength, out int rowCount) { var bufferSize = 1024; var probeSize = 16; @@ -29,7 +29,7 @@ public static void Analyze(Stream stream, char[] separators, Encoding fallbackEn { separatorInfos[i] = new SeparatorInfo { - Buffer = new CsvParser(separators[i], autodetectEncoding) + Buffer = new CsvParser(separators[i], autodetectEncoding, quoteChar) }; } @@ -191,4 +191,4 @@ private sealed class SeparatorInfo public CsvParser Buffer { get; set; } } -} +} \ No newline at end of file diff --git a/src/ExcelDataReader/Core/CsvFormat/CsvParser.cs b/src/ExcelDataReader/Core/CsvFormat/CsvParser.cs index ea2c43f9..a0fa2a17 100644 --- a/src/ExcelDataReader/Core/CsvFormat/CsvParser.cs +++ b/src/ExcelDataReader/Core/CsvFormat/CsvParser.cs @@ -7,10 +7,10 @@ namespace ExcelDataReader.Core.CsvFormat; /// internal sealed class CsvParser { - public CsvParser(char separator, Encoding encoding) + public CsvParser(char separator, Encoding encoding, char? quoteChar = null) { Separator = separator; - QuoteChar = '"'; + QuoteChar = quoteChar; Decoder = encoding.GetDecoder(); Decoder.Fallback = new DecoderExceptionFallback(); @@ -34,7 +34,7 @@ private enum CsvState private CsvState State { get; set; } - private char QuoteChar { get; } + private char? QuoteChar { get; } private int TrailingWhitespaceCount { get; set; } @@ -93,21 +93,27 @@ private void ParseChar(char c, int bytesUsed) case CsvState.PreValue: parsed = ReadPreValue(c, bytesUsed); break; + case CsvState.Value: parsed = ReadValue(c, bytesUsed); break; + case CsvState.QuotedValue: parsed = ReadQuotedValue(c, bytesUsed); break; + case CsvState.QuotedValueQuote: parsed = ReadQuotedValueQuote(c, bytesUsed); break; + case CsvState.Separator: parsed = ReadSeparator(c, bytesUsed); break; + case CsvState.Linebreak: parsed = ReadLinebreak(c, bytesUsed); break; + default: throw new InvalidOperationException("Unhandled parser state: " + State); } @@ -120,7 +126,7 @@ private bool ReadPreValue(char c, int bytesUsed) { return true; } - else if (c == QuoteChar) + else if (QuoteChar.HasValue && c == QuoteChar.Value) { State = CsvState.QuotedValue; return true; @@ -182,7 +188,7 @@ private bool ReadValue(char c, int bytesUsed) private bool ReadQuotedValue(char c, int bytesUsed) { - if (c == QuoteChar) + if (QuoteChar.HasValue && c == QuoteChar.Value) { State = CsvState.QuotedValueQuote; return true; @@ -196,7 +202,7 @@ private bool ReadQuotedValue(char c, int bytesUsed) private bool ReadQuotedValueQuote(char c, int bytesUsed) { - if (c == QuoteChar) + if (QuoteChar.HasValue && c == QuoteChar.Value) { // Is escaped quote ValueResult.Append(c); @@ -244,7 +250,7 @@ private bool ReadLinebreak(char c, int bytesUsed) private void AddValueToRow() { - RowResult.Add(ValueResult.ToString(0, ValueResult.Length - TrailingWhitespaceCount)); + RowResult.Add(ValueResult.ToString(0, ValueResult.Length - TrailingWhitespaceCount)); ValueResult = new StringBuilder(); TrailingWhitespaceCount = 0; } @@ -269,4 +275,4 @@ private bool IsWhitespace(char c) return false; } -} +} \ No newline at end of file diff --git a/src/ExcelDataReader/Core/CsvFormat/CsvWorkbook.cs b/src/ExcelDataReader/Core/CsvFormat/CsvWorkbook.cs index f1af684d..3b0f0f05 100644 --- a/src/ExcelDataReader/Core/CsvFormat/CsvWorkbook.cs +++ b/src/ExcelDataReader/Core/CsvFormat/CsvWorkbook.cs @@ -3,7 +3,7 @@ namespace ExcelDataReader.Core.CsvFormat; -internal sealed class CsvWorkbook(Stream stream, Encoding encoding, char[] autodetectSeparators, int analyzeInitialCsvRows) : IWorkbook +internal sealed class CsvWorkbook(Stream stream, Encoding encoding, char[] autodetectSeparators, int analyzeInitialCsvRows, char? quoteChar = null) : IWorkbook { public int ResultsCount => 1; @@ -13,17 +13,19 @@ internal sealed class CsvWorkbook(Stream stream, Encoding encoding, char[] autod public Encoding Encoding { get; } = encoding; + public char? QuoteChar { get; } = quoteChar; + public char[] AutodetectSeparators { get; } = autodetectSeparators; public int AnalyzeInitialCsvRows { get; } = analyzeInitialCsvRows; public IEnumerable ReadWorksheets() { - yield return new CsvWorksheet(Stream, Encoding, AutodetectSeparators, AnalyzeInitialCsvRows); + yield return new CsvWorksheet(Stream, Encoding, AutodetectSeparators, AnalyzeInitialCsvRows, QuoteChar); } public NumberFormatString GetNumberFormatString(int index) { return null; } -} +} \ No newline at end of file diff --git a/src/ExcelDataReader/Core/CsvFormat/CsvWorksheet.cs b/src/ExcelDataReader/Core/CsvFormat/CsvWorksheet.cs index 65141453..3beedaac 100644 --- a/src/ExcelDataReader/Core/CsvFormat/CsvWorksheet.cs +++ b/src/ExcelDataReader/Core/CsvFormat/CsvWorksheet.cs @@ -4,14 +4,15 @@ namespace ExcelDataReader.Core.CsvFormat; internal sealed class CsvWorksheet : IWorksheet { - public CsvWorksheet(Stream stream, Encoding fallbackEncoding, char[] autodetectSeparators, int analyzeInitialCsvRows) + public CsvWorksheet(Stream stream, Encoding fallbackEncoding, char[] autodetectSeparators, int analyzeInitialCsvRows, char? quoteChar = null) { Stream = stream; + QuoteChar = quoteChar; Stream.Seek(0, SeekOrigin.Begin); try { // Try as UTF-8 first, or use BOM if present - CsvAnalyzer.Analyze(Stream, autodetectSeparators, Encoding.UTF8, analyzeInitialCsvRows, out var fieldCount, out var separator, out var encoding, out var bomLength, out var rowCount); + CsvAnalyzer.Analyze(Stream, autodetectSeparators, Encoding.UTF8, analyzeInitialCsvRows, quoteChar, out var fieldCount, out var separator, out var encoding, out var bomLength, out var rowCount); FieldCount = fieldCount; AnalyzedRowCount = rowCount; AnalyzedPartial = analyzeInitialCsvRows > 0; @@ -24,7 +25,7 @@ public CsvWorksheet(Stream stream, Encoding fallbackEncoding, char[] autodetectS // If cannot parse as UTF-8, try fallback encoding Stream.Seek(0, SeekOrigin.Begin); - CsvAnalyzer.Analyze(Stream, autodetectSeparators, fallbackEncoding, analyzeInitialCsvRows, out var fieldCount, out var separator, out var encoding, out var bomLength, out var rowCount); + CsvAnalyzer.Analyze(Stream, autodetectSeparators, fallbackEncoding, analyzeInitialCsvRows, quoteChar, out var fieldCount, out var separator, out var encoding, out var bomLength, out var rowCount); FieldCount = fieldCount; AnalyzedRowCount = rowCount; AnalyzedPartial = analyzeInitialCsvRows > 0; @@ -67,6 +68,8 @@ public int RowCount public Encoding Encoding { get; } + public char? QuoteChar { get; } + public char Separator { get; } public Column[] ColumnWidths => null; @@ -82,7 +85,7 @@ public IEnumerable ReadRows() var bufferSize = 1024; var buffer = new byte[bufferSize]; var rowIndex = 0; - var csv = new CsvParser(Separator, Encoding); + var csv = new CsvParser(Separator, Encoding, QuoteChar); var skipBomBytes = BomLength; Stream.Seek(0, SeekOrigin.Begin); @@ -124,4 +127,4 @@ private static IEnumerable GetReaderRows(int rowIndex, List> r rowIndex++; } } -} +} \ No newline at end of file diff --git a/src/ExcelDataReader/ExcelCsvReader.cs b/src/ExcelDataReader/ExcelCsvReader.cs index a6fb7f02..243f2c01 100644 --- a/src/ExcelDataReader/ExcelCsvReader.cs +++ b/src/ExcelDataReader/ExcelCsvReader.cs @@ -5,9 +5,9 @@ namespace ExcelDataReader; internal sealed class ExcelCsvReader : ExcelDataReader { - public ExcelCsvReader(Stream stream, Encoding fallbackEncoding, char[] autodetectSeparators, int analyzeInitialCsvRows) + public ExcelCsvReader(Stream stream, Encoding fallbackEncoding, char[] autodetectSeparators, int analyzeInitialCsvRows, char? quoteChar = null) { - Workbook = new CsvWorkbook(stream, fallbackEncoding, autodetectSeparators, analyzeInitialCsvRows); + Workbook = new CsvWorkbook(stream, fallbackEncoding, autodetectSeparators, analyzeInitialCsvRows, quoteChar); // By default, the data reader is positioned on the first result. Reset(); @@ -19,4 +19,4 @@ public override void Close() Workbook?.Stream?.Dispose(); Workbook = null; } -} +} \ No newline at end of file diff --git a/src/ExcelDataReader/ExcelReaderConfiguration.cs b/src/ExcelDataReader/ExcelReaderConfiguration.cs index af30e766..26fbc691 100644 --- a/src/ExcelDataReader/ExcelReaderConfiguration.cs +++ b/src/ExcelDataReader/ExcelReaderConfiguration.cs @@ -8,7 +8,7 @@ namespace ExcelDataReader; public class ExcelReaderConfiguration { /// - /// Gets or sets a value indicating the encoding to use when the input XLS lacks a CodePage record, + /// Gets or sets a value indicating the encoding to use when the input XLS lacks a CodePage record, /// or when the input CSV lacks a BOM and does not parse as UTF8. Default: cp1252. (XLS BIFF2-5 and CSV only). /// public Encoding FallbackEncoding { get; set; } = Encoding.GetEncoding(1252); @@ -23,6 +23,11 @@ public class ExcelReaderConfiguration /// public char[] AutodetectSeparators { get; set; } = [',', ';', '\t', '|', '#']; + /// + /// Gets or sets a QuoteCharacter for CSV (Default '"'). + /// + public char? QuoteChar { get; set; } = '"'; + /// /// Gets or sets a value indicating whether to leave the stream open after the IExcelDataReader object is disposed. Default: false. /// @@ -34,4 +39,4 @@ public class ExcelReaderConfiguration /// Default: 0 - analyzes the entire file (CSV only, has no effect on other formats). /// public int AnalyzeInitialCsvRows { get; set; } -} +} \ No newline at end of file diff --git a/src/ExcelDataReader/ExcelReaderFactory.cs b/src/ExcelDataReader/ExcelReaderFactory.cs index ed8eb1fd..aad18d40 100644 --- a/src/ExcelDataReader/ExcelReaderFactory.cs +++ b/src/ExcelDataReader/ExcelReaderFactory.cs @@ -165,7 +165,7 @@ public static IExcelDataReader CreateCsvReader(Stream fileStream, ExcelReaderCon fileStream = new LeaveOpenStream(fileStream); } - return new ExcelCsvReader(fileStream, configuration.FallbackEncoding, configuration.AutodetectSeparators, configuration.AnalyzeInitialCsvRows); + return new ExcelCsvReader(fileStream, configuration.FallbackEncoding, configuration.AutodetectSeparators, configuration.AnalyzeInitialCsvRows, configuration.QuoteChar); } private static bool TryGetWorkbook(Stream fileStream, CompoundDocument document, out Stream stream)