From 8dca60f4ee0d3a305e2f65272b0376e5f172ce71 Mon Sep 17 00:00:00 2001 From: Johan Appelgren Date: Wed, 27 May 2026 09:47:25 +0200 Subject: [PATCH 1/8] Add cross-platform console sample app --- ExcelDataReader.slnx | 1 + .../ExcelDataReader.ConsoleSample.csproj | 20 ++ src/ExcelDataReader.ConsoleSample/Program.cs | 275 ++++++++++++++++++ .../packages.lock.json | 50 ++++ 4 files changed, 346 insertions(+) create mode 100644 src/ExcelDataReader.ConsoleSample/ExcelDataReader.ConsoleSample.csproj create mode 100644 src/ExcelDataReader.ConsoleSample/Program.cs create mode 100644 src/ExcelDataReader.ConsoleSample/packages.lock.json diff --git a/ExcelDataReader.slnx b/ExcelDataReader.slnx index c96a1563..609e67d9 100644 --- a/ExcelDataReader.slnx +++ b/ExcelDataReader.slnx @@ -4,6 +4,7 @@ + diff --git a/src/ExcelDataReader.ConsoleSample/ExcelDataReader.ConsoleSample.csproj b/src/ExcelDataReader.ConsoleSample/ExcelDataReader.ConsoleSample.csproj new file mode 100644 index 00000000..d50fdeb5 --- /dev/null +++ b/src/ExcelDataReader.ConsoleSample/ExcelDataReader.ConsoleSample.csproj @@ -0,0 +1,20 @@ + + + + readersample + Exe + net10.0 + enable + + + + + + + + + + + + + diff --git a/src/ExcelDataReader.ConsoleSample/Program.cs b/src/ExcelDataReader.ConsoleSample/Program.cs new file mode 100644 index 00000000..22fff736 --- /dev/null +++ b/src/ExcelDataReader.ConsoleSample/Program.cs @@ -0,0 +1,275 @@ +using System.CommandLine; +using System.Data; +using System.Diagnostics; +using System.Text; +using ExcelDataReader; +using Spectre.Console; + +Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); + +var rootCommand = new RootCommand("ExcelDataReader sample console application"); +rootCommand.Subcommands.Add(BuildExcelCommand()); +rootCommand.Subcommands.Add(BuildCsvCommand()); +return rootCommand.Parse(args).Invoke(); + +// ---- excel subcommand ------------------------------------------------------- +static Command BuildExcelCommand() +{ + var fileArg = new Argument("file") { Description = "Path to the Excel file (XLS, XLSX, XLSB)" }; + var sheetNameOpt = new Option("--sheet-name", ["-n"]) { Description = "Filter by sheet name (repeatable; names may contain commas)", AllowMultipleArgumentsPerToken = false }; + var sheetIndexOpt = new Option("--sheet-index", ["-i"]) { Description = "Filter by 1-based sheet index, comma-separated or repeatable (e.g. 2,5,7)", AllowMultipleArgumentsPerToken = false }; + var noHeaderOpt = new Option("--no-header", ["-H"]) { Description = "Don't treat first row as column names" }; + var noFillMergedOpt = new Option("--no-fill-merged") { Description = "Don't fill merged cell values" }; + var singlePassOpt = new Option("--single-pass") { Description = "Enable single pass mode (skips pre-scan for row/column counts)" }; + var outputOpt = new Option("--output", ["-o"]) { Description = "Data output format: table, csv, tsv (default: no data output, stats only)", DefaultValueFactory = _ => OutputFormat.None }; + var passwordOpt = new Option("--password", ["-p"]) { Description = "Password for protected workbooks" }; + var encodingOpt = new Option("--encoding", ["-e"]) { Description = "Fallback encoding for XLS BIFF2-5 (default: windows-1252)", DefaultValueFactory = _ => "windows-1252" }; + + var cmd = new Command("excel", "Read XLS, XLSX, or XLSB files"); + cmd.Arguments.Add(fileArg); + cmd.Options.Add(sheetNameOpt); + cmd.Options.Add(sheetIndexOpt); + cmd.Options.Add(noHeaderOpt); + cmd.Options.Add(noFillMergedOpt); + cmd.Options.Add(singlePassOpt); + cmd.Options.Add(outputOpt); + cmd.Options.Add(passwordOpt); + cmd.Options.Add(encodingOpt); + + cmd.SetAction(parseResult => + { + var file = parseResult.GetValue(fileArg)!; + var sheetNames = parseResult.GetValue(sheetNameOpt) ?? []; + var sheetIndexTokens = parseResult.GetValue(sheetIndexOpt) ?? []; + var noHeader = parseResult.GetValue(noHeaderOpt); + var noFillMerged = parseResult.GetValue(noFillMergedOpt); + var singlePass = parseResult.GetValue(singlePassOpt); + var output = parseResult.GetValue(outputOpt); + var password = parseResult.GetValue(passwordOpt); + var encodingName = parseResult.GetValue(encodingOpt)!; + + // Expand comma-separated index tokens into a set of 1-based indices. + var nameSet = new HashSet(sheetNames, StringComparer.Ordinal); + var indexSet = new HashSet(); + foreach (var token in sheetIndexTokens) + { + foreach (var part in token.Split(',')) + { + if (!int.TryParse(part.Trim(), out int idx) || idx < 1) + { + Console.Error.WriteLine($"Invalid sheet index: '{part.Trim()}' (must be a positive integer)"); + return; + } + + indexSet.Add(idx); + } + } + + bool hasFilter = nameSet.Count > 0 || indexSet.Count > 0; + + var memBefore = Process.GetCurrentProcess().WorkingSet64; + var sw = Stopwatch.StartNew(); + + using var stream = file.OpenRead(); + using var reader = ExcelReaderFactory.CreateReader(stream, new ExcelReaderConfiguration + { + Password = password, + FallbackEncoding = Encoding.GetEncoding(encodingName), + SinglePassMode = singlePass, + }); + + var openMs = sw.ElapsedMilliseconds; + + var ds = reader.AsDataSet(new ExcelDataSetConfiguration + { + UseColumnDataType = false, + FilterSheet = (tableReader, sheetIndex) => + { + if (!hasFilter) + return true; + + // sheetIndex is 0-based; expose as 1-based to the user. + return nameSet.Contains(tableReader.Name) || indexSet.Contains(sheetIndex + 1); + }, + ConfigureDataTable = _ => new ExcelDataTableConfiguration + { + UseHeaderRow = !noHeader, + FillMergedCellsValue = !noFillMerged, + } + }); + + var readMs = sw.ElapsedMilliseconds - openMs; + var memAfter = Process.GetCurrentProcess().WorkingSet64; + + PrintStats(openMs, readMs, ds, memBefore, memAfter); + + if (output != OutputFormat.None) + RenderSheets(ds, output); + }); + + return cmd; +} + +// ---- csv subcommand --------------------------------------------------------- +static Command BuildCsvCommand() +{ + var fileArg = new Argument("file") { Description = "Path to the CSV file" }; + var noHeaderOpt = new Option("--no-header", ["-H"]) { Description = "Don't treat first row as column names" }; + var outputOpt = new Option("--output", ["-o"]) { Description = "Data output format: table, csv, tsv (default: no data output, stats only)", DefaultValueFactory = _ => OutputFormat.None }; + var encodingOpt = new Option("--encoding", ["-e"]) { Description = "Fallback encoding when no BOM / not UTF-8 (default: windows-1252)", DefaultValueFactory = _ => "windows-1252" }; + var noTrimOpt = new Option("--no-trim") { Description = "Don't trim whitespace in values" }; + var separatorsOpt = new Option("--separators") { Description = "Separator candidates, e.g. \",;\" -- use \\t for TAB (default: , ; TAB | #)" }; + var quoteCharOpt = new Option("--quote-char") { Description = "Quote character (default: \")" }; + var escapeCharOpt = new Option("--escape-char") { Description = "Escape character for quoted fields (default: disabled)" }; + + var cmd = new Command("csv", "Read CSV files"); + cmd.Arguments.Add(fileArg); + cmd.Options.Add(noHeaderOpt); + cmd.Options.Add(outputOpt); + cmd.Options.Add(encodingOpt); + cmd.Options.Add(noTrimOpt); + cmd.Options.Add(separatorsOpt); + cmd.Options.Add(quoteCharOpt); + cmd.Options.Add(escapeCharOpt); + + cmd.SetAction(parseResult => + { + var file = parseResult.GetValue(fileArg)!; + var noHeader = parseResult.GetValue(noHeaderOpt); + var output = parseResult.GetValue(outputOpt); + var encodingName = parseResult.GetValue(encodingOpt)!; + var noTrim = parseResult.GetValue(noTrimOpt); + var separatorsStr = parseResult.GetValue(separatorsOpt); + var quoteCharStr = parseResult.GetValue(quoteCharOpt); + var escapeCharStr = parseResult.GetValue(escapeCharOpt); + + var config = new ExcelReaderConfiguration + { + FallbackEncoding = Encoding.GetEncoding(encodingName), + TrimWhiteSpace = !noTrim, + QuoteChar = quoteCharStr?.Length > 0 ? quoteCharStr[0] : '"', + EscapeChar = escapeCharStr?.Length > 0 ? escapeCharStr[0] : null, + }; + if (separatorsStr is not null) + config.AutodetectSeparators = separatorsStr.Replace("\\t", "\t").ToCharArray(); + + var memBefore = Process.GetCurrentProcess().WorkingSet64; + var sw = Stopwatch.StartNew(); + + using var stream = file.OpenRead(); + using var reader = ExcelReaderFactory.CreateCsvReader(stream, config); + + var openMs = sw.ElapsedMilliseconds; + + var ds = reader.AsDataSet(new ExcelDataSetConfiguration + { + UseColumnDataType = false, + ConfigureDataTable = _ => new ExcelDataTableConfiguration + { + UseHeaderRow = !noHeader, + } + }); + + var readMs = sw.ElapsedMilliseconds - openMs; + var memAfter = Process.GetCurrentProcess().WorkingSet64; + + PrintStats(openMs, readMs, ds, memBefore, memAfter); + + if (output != OutputFormat.None) + RenderSheets(ds, output); + }); + + return cmd; +} + +// ---- helpers ---------------------------------------------------------------- +static void PrintStats(long openMs, long readMs, DataSet ds, long memBefore, long memAfter) +{ + long totalRows = 0; + int maxCols = 0; + + foreach (DataTable dt in ds.Tables) + { + totalRows += dt.Rows.Count; + if (dt.Columns.Count > maxCols) + maxCols = dt.Columns.Count; + } + + long totalCells = totalRows * maxCols; + double cellsPerSec = readMs > 0 ? totalCells * 1000.0 / readMs : double.PositiveInfinity; + long memDeltaMb = (memAfter - memBefore) / (1024 * 1024); + + Console.Error.WriteLine($"Open: {openMs,6:N0} ms"); + Console.Error.Write($"Read: {readMs,6:N0} ms ({totalRows:N0} rows x {maxCols} cols"); + if (ds.Tables.Count > 1) + { + Console.Error.WriteLine($" across {ds.Tables.Count} sheets)"); + foreach (DataTable dt in ds.Tables) + Console.Error.WriteLine($" {dt.TableName}: {dt.Rows.Count:N0} rows x {dt.Columns.Count} cols"); + } + else + { + Console.Error.WriteLine(")"); + } + + Console.Error.WriteLine($"Total: {openMs + readMs,6:N0} ms (~{cellsPerSec / 1_000_000:F1}M cells/sec)"); + Console.Error.WriteLine($"Memory: {(memDeltaMb >= 0 ? "+" : string.Empty)}{memDeltaMb} MB"); +} + +static void RenderSheets(DataSet ds, OutputFormat output) +{ + bool multiSheet = ds.Tables.Count > 1; + bool first = true; + + foreach (DataTable dt in ds.Tables) + { + if (output == OutputFormat.Table) + { + if (multiSheet) + AnsiConsole.MarkupLine($"[bold]{Markup.Escape(dt.TableName)}[/]"); + RenderSpectreTable(dt); + } + else + { + if (!first) + Console.WriteLine(); + if (multiSheet) + Console.WriteLine($"# Sheet: {dt.TableName}"); + WriteSeparated(dt, output == OutputFormat.Csv ? ',' : '\t'); + } + + first = false; + } +} + +static void RenderSpectreTable(DataTable dt) +{ + var table = new Table(); + foreach (DataColumn col in dt.Columns) + table.AddColumn(Markup.Escape(col.ColumnName)); + + foreach (DataRow row in dt.Rows) + table.AddRow(row.ItemArray.Select(v => Markup.Escape(v?.ToString() ?? string.Empty)).ToArray()); + + AnsiConsole.Write(table); +} + +static void WriteSeparated(DataTable dt, char sep) +{ + static string Escape(string s, char sep) => + s.Contains(sep) || s.Contains('"') || s.Contains('\n') + ? $"\"{s.Replace("\"", "\"\"")}\"" + : s; + + foreach (DataRow row in dt.Rows) + Console.WriteLine(string.Join(sep, row.ItemArray.Select(v => Escape(v?.ToString() ?? string.Empty, sep)))); +} + +#pragma warning disable SA1649 // File name should match first type name (top-level statements use implicit Program class) +internal enum OutputFormat +{ + None, + Table, + Csv, + Tsv, +} \ No newline at end of file diff --git a/src/ExcelDataReader.ConsoleSample/packages.lock.json b/src/ExcelDataReader.ConsoleSample/packages.lock.json new file mode 100644 index 00000000..c1578ab5 --- /dev/null +++ b/src/ExcelDataReader.ConsoleSample/packages.lock.json @@ -0,0 +1,50 @@ +{ + "version": 1, + "dependencies": { + "net10.0": { + "Spectre.Console": { + "type": "Direct", + "requested": "[0.55.2, )", + "resolved": "0.55.2", + "contentHash": "yc0g12YJpmZHofGJLTQX02/uqfzJ6XHRx6pe9rANNjleQptXMHPxD9KI2/DWkXiJWddc+4oizEdGox15c89hhw==", + "dependencies": { + "Spectre.Console.Ansi": "0.55.2" + } + }, + "StyleCop.Analyzers": { + "type": "Direct", + "requested": "[1.2.0-beta.556, )", + "resolved": "1.2.0-beta.556", + "contentHash": "llRPgmA1fhC0I0QyFLEcjvtM2239QzKr/tcnbsjArLMJxJlu0AA5G7Fft0OI30pHF3MW63Gf4aSSsjc5m82J1Q==", + "dependencies": { + "StyleCop.Analyzers.Unstable": "1.2.0.556" + } + }, + "System.CommandLine": { + "type": "Direct", + "requested": "[2.0.8, )", + "resolved": "2.0.8", + "contentHash": "FbpgF8p/ClXnoXEWLjQB34kNh5rsLewEgIgLyVzLDucAOQ4cNs7ec9Cam7gdKPruSb6zp4Mx8htZGTL4/5PJPg==" + }, + "Spectre.Console.Ansi": { + "type": "Transitive", + "resolved": "0.55.2", + "contentHash": "tN3KEJQQZwLW9HJnG5YTSAxfBwxsmg1wVuGvhGG2rM+pbl+NGaHU8pDcNofPlOXlgpkbpBMxI5t9o9qPfy3sHQ==" + }, + "StyleCop.Analyzers.Unstable": { + "type": "Transitive", + "resolved": "1.2.0.556", + "contentHash": "zvn9Mqs/ox/83cpYPignI8hJEM2A93s2HkHs8HYMOAQW0PkampyoErAiIyKxgTLqbbad29HX/shv/6LGSjPJNQ==" + }, + "exceldatareader": { + "type": "Project" + }, + "exceldatareader.dataset": { + "type": "Project", + "dependencies": { + "ExcelDataReader": "[3.9.0, )" + } + } + } + } +} \ No newline at end of file From 221c802842c232bd7884a3ca3781ffedfb00a298 Mon Sep 17 00:00:00 2001 From: Johan Appelgren Date: Wed, 27 May 2026 11:18:46 +0200 Subject: [PATCH 2/8] Configure NativeAOT publish for console sample --- Directory.build.props | 2 + .../ExcelDataReader.ConsoleSample.csproj | 1 + .../packages.lock.json | 50 ------------------- 3 files changed, 3 insertions(+), 50 deletions(-) delete mode 100644 src/ExcelDataReader.ConsoleSample/packages.lock.json diff --git a/Directory.build.props b/Directory.build.props index 612d02f4..bc8f5d94 100644 --- a/Directory.build.props +++ b/Directory.build.props @@ -6,6 +6,8 @@ true $(MSBuildThisFileDirectory)artifacts true + + false diff --git a/src/ExcelDataReader.ConsoleSample/ExcelDataReader.ConsoleSample.csproj b/src/ExcelDataReader.ConsoleSample/ExcelDataReader.ConsoleSample.csproj index d50fdeb5..e7847ea0 100644 --- a/src/ExcelDataReader.ConsoleSample/ExcelDataReader.ConsoleSample.csproj +++ b/src/ExcelDataReader.ConsoleSample/ExcelDataReader.ConsoleSample.csproj @@ -5,6 +5,7 @@ Exe net10.0 enable + true diff --git a/src/ExcelDataReader.ConsoleSample/packages.lock.json b/src/ExcelDataReader.ConsoleSample/packages.lock.json deleted file mode 100644 index c1578ab5..00000000 --- a/src/ExcelDataReader.ConsoleSample/packages.lock.json +++ /dev/null @@ -1,50 +0,0 @@ -{ - "version": 1, - "dependencies": { - "net10.0": { - "Spectre.Console": { - "type": "Direct", - "requested": "[0.55.2, )", - "resolved": "0.55.2", - "contentHash": "yc0g12YJpmZHofGJLTQX02/uqfzJ6XHRx6pe9rANNjleQptXMHPxD9KI2/DWkXiJWddc+4oizEdGox15c89hhw==", - "dependencies": { - "Spectre.Console.Ansi": "0.55.2" - } - }, - "StyleCop.Analyzers": { - "type": "Direct", - "requested": "[1.2.0-beta.556, )", - "resolved": "1.2.0-beta.556", - "contentHash": "llRPgmA1fhC0I0QyFLEcjvtM2239QzKr/tcnbsjArLMJxJlu0AA5G7Fft0OI30pHF3MW63Gf4aSSsjc5m82J1Q==", - "dependencies": { - "StyleCop.Analyzers.Unstable": "1.2.0.556" - } - }, - "System.CommandLine": { - "type": "Direct", - "requested": "[2.0.8, )", - "resolved": "2.0.8", - "contentHash": "FbpgF8p/ClXnoXEWLjQB34kNh5rsLewEgIgLyVzLDucAOQ4cNs7ec9Cam7gdKPruSb6zp4Mx8htZGTL4/5PJPg==" - }, - "Spectre.Console.Ansi": { - "type": "Transitive", - "resolved": "0.55.2", - "contentHash": "tN3KEJQQZwLW9HJnG5YTSAxfBwxsmg1wVuGvhGG2rM+pbl+NGaHU8pDcNofPlOXlgpkbpBMxI5t9o9qPfy3sHQ==" - }, - "StyleCop.Analyzers.Unstable": { - "type": "Transitive", - "resolved": "1.2.0.556", - "contentHash": "zvn9Mqs/ox/83cpYPignI8hJEM2A93s2HkHs8HYMOAQW0PkampyoErAiIyKxgTLqbbad29HX/shv/6LGSjPJNQ==" - }, - "exceldatareader": { - "type": "Project" - }, - "exceldatareader.dataset": { - "type": "Project", - "dependencies": { - "ExcelDataReader": "[3.9.0, )" - } - } - } - } -} \ No newline at end of file From a192a0af716e72d59bd99b1f36c3d957f6326c2b Mon Sep 17 00:00:00 2001 From: Johan Appelgren Date: Wed, 27 May 2026 11:21:44 +0200 Subject: [PATCH 3/8] Fix Directory.build.props casing for Linux compatibility --- Directory.build.props => Directory.Build.props | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Directory.build.props => Directory.Build.props (100%) diff --git a/Directory.build.props b/Directory.Build.props similarity index 100% rename from Directory.build.props rename to Directory.Build.props From fd603540b2cdb4d0136aae1885322755e47edde3 Mon Sep 17 00:00:00 2001 From: Johan Appelgren Date: Wed, 27 May 2026 11:24:07 +0200 Subject: [PATCH 4/8] Fix IL2072 AOT trim warning in FixDataTypes --- src/ExcelDataReader.DataSet/ExcelDataReaderExtensions.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/ExcelDataReader.DataSet/ExcelDataReaderExtensions.cs b/src/ExcelDataReader.DataSet/ExcelDataReaderExtensions.cs index 4501accc..60a02d12 100644 --- a/src/ExcelDataReader.DataSet/ExcelDataReaderExtensions.cs +++ b/src/ExcelDataReader.DataSet/ExcelDataReaderExtensions.cs @@ -1,4 +1,5 @@ using System.Data; +using System.Diagnostics.CodeAnalysis; using System.Globalization; namespace ExcelDataReader; @@ -257,6 +258,11 @@ private static bool IsEmptyRow(IExcelDataReader reader, ExcelDataTableConfigurat return true; } + // DataColumn.DataType setter requires [DynamicallyAccessedMembers] but the type comes from GetType() on + // values written by ExcelDataReader (string, double, DateTime, etc.) — always available at runtime. +#if NET5_0_OR_GREATER + [System.Diagnostics.CodeAnalysis.UnconditionalSuppressMessage("Trimming", "IL2072", Justification = "Types returned by GetType() on ExcelDataReader output values are always fully available at runtime.")] +#endif private static void FixDataTypes(DataSet dataset) { var tables = new List(dataset.Tables.Count); From 65fc0564a2e07562b1b855cad72d40ef3a5f95b8 Mon Sep 17 00:00:00 2001 From: Johan Appelgren Date: Wed, 27 May 2026 11:55:19 +0200 Subject: [PATCH 5/8] Change --no-fill-merged to opt-in --fill-merged in console sample --- src/ExcelDataReader.ConsoleSample/Program.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ExcelDataReader.ConsoleSample/Program.cs b/src/ExcelDataReader.ConsoleSample/Program.cs index 22fff736..b43c8253 100644 --- a/src/ExcelDataReader.ConsoleSample/Program.cs +++ b/src/ExcelDataReader.ConsoleSample/Program.cs @@ -19,7 +19,7 @@ static Command BuildExcelCommand() var sheetNameOpt = new Option("--sheet-name", ["-n"]) { Description = "Filter by sheet name (repeatable; names may contain commas)", AllowMultipleArgumentsPerToken = false }; var sheetIndexOpt = new Option("--sheet-index", ["-i"]) { Description = "Filter by 1-based sheet index, comma-separated or repeatable (e.g. 2,5,7)", AllowMultipleArgumentsPerToken = false }; var noHeaderOpt = new Option("--no-header", ["-H"]) { Description = "Don't treat first row as column names" }; - var noFillMergedOpt = new Option("--no-fill-merged") { Description = "Don't fill merged cell values" }; + var fillMergedOpt = new Option("--fill-merged") { Description = "Fill merged cell values across the merged range" }; var singlePassOpt = new Option("--single-pass") { Description = "Enable single pass mode (skips pre-scan for row/column counts)" }; var outputOpt = new Option("--output", ["-o"]) { Description = "Data output format: table, csv, tsv (default: no data output, stats only)", DefaultValueFactory = _ => OutputFormat.None }; var passwordOpt = new Option("--password", ["-p"]) { Description = "Password for protected workbooks" }; @@ -30,7 +30,7 @@ static Command BuildExcelCommand() cmd.Options.Add(sheetNameOpt); cmd.Options.Add(sheetIndexOpt); cmd.Options.Add(noHeaderOpt); - cmd.Options.Add(noFillMergedOpt); + cmd.Options.Add(fillMergedOpt); cmd.Options.Add(singlePassOpt); cmd.Options.Add(outputOpt); cmd.Options.Add(passwordOpt); @@ -42,7 +42,7 @@ static Command BuildExcelCommand() var sheetNames = parseResult.GetValue(sheetNameOpt) ?? []; var sheetIndexTokens = parseResult.GetValue(sheetIndexOpt) ?? []; var noHeader = parseResult.GetValue(noHeaderOpt); - var noFillMerged = parseResult.GetValue(noFillMergedOpt); + var noFillMerged = !parseResult.GetValue(fillMergedOpt); var singlePass = parseResult.GetValue(singlePassOpt); var output = parseResult.GetValue(outputOpt); var password = parseResult.GetValue(passwordOpt); From 752ea7905eaa105ad5bab001eed4c73ff9a145cf Mon Sep 17 00:00:00 2001 From: Johan Appelgren Date: Wed, 27 May 2026 13:06:04 +0200 Subject: [PATCH 6/8] Make IExcelDataReader the default read path, add --dataset opt-in Use IExcelDataReader directly by default for lower memory usage. The AsDataSet extension is available via --dataset (and is implied by --fill-merged). Raw mode supports all --output formats: csv/tsv streams row-by-row, table buffers as string arrays. Table output is capped at 100 rows to prevent hangs on large files; a note on stderr directs users to --output csv for full data. --- src/ExcelDataReader.ConsoleSample/Program.cs | 268 ++++++++++++++++--- 1 file changed, 232 insertions(+), 36 deletions(-) diff --git a/src/ExcelDataReader.ConsoleSample/Program.cs b/src/ExcelDataReader.ConsoleSample/Program.cs index b43c8253..b70bf947 100644 --- a/src/ExcelDataReader.ConsoleSample/Program.cs +++ b/src/ExcelDataReader.ConsoleSample/Program.cs @@ -19,11 +19,12 @@ static Command BuildExcelCommand() var sheetNameOpt = new Option("--sheet-name", ["-n"]) { Description = "Filter by sheet name (repeatable; names may contain commas)", AllowMultipleArgumentsPerToken = false }; var sheetIndexOpt = new Option("--sheet-index", ["-i"]) { Description = "Filter by 1-based sheet index, comma-separated or repeatable (e.g. 2,5,7)", AllowMultipleArgumentsPerToken = false }; var noHeaderOpt = new Option("--no-header", ["-H"]) { Description = "Don't treat first row as column names" }; - var fillMergedOpt = new Option("--fill-merged") { Description = "Fill merged cell values across the merged range" }; + var fillMergedOpt = new Option("--fill-merged") { Description = "Fill merged cell values across the merged range (implies --dataset)" }; var singlePassOpt = new Option("--single-pass") { Description = "Enable single pass mode (skips pre-scan for row/column counts)" }; var outputOpt = new Option("--output", ["-o"]) { Description = "Data output format: table, csv, tsv (default: no data output, stats only)", DefaultValueFactory = _ => OutputFormat.None }; var passwordOpt = new Option("--password", ["-p"]) { Description = "Password for protected workbooks" }; var encodingOpt = new Option("--encoding", ["-e"]) { Description = "Fallback encoding for XLS BIFF2-5 (default: windows-1252)", DefaultValueFactory = _ => "windows-1252" }; + var dataSetOpt = new Option("--dataset") { Description = "Use AsDataSet extension (loads all data into a DataSet in memory)" }; var cmd = new Command("excel", "Read XLS, XLSX, or XLSB files"); cmd.Arguments.Add(fileArg); @@ -35,6 +36,7 @@ static Command BuildExcelCommand() cmd.Options.Add(outputOpt); cmd.Options.Add(passwordOpt); cmd.Options.Add(encodingOpt); + cmd.Options.Add(dataSetOpt); cmd.SetAction(parseResult => { @@ -42,11 +44,12 @@ static Command BuildExcelCommand() var sheetNames = parseResult.GetValue(sheetNameOpt) ?? []; var sheetIndexTokens = parseResult.GetValue(sheetIndexOpt) ?? []; var noHeader = parseResult.GetValue(noHeaderOpt); - var noFillMerged = !parseResult.GetValue(fillMergedOpt); + var fillMerged = parseResult.GetValue(fillMergedOpt); var singlePass = parseResult.GetValue(singlePassOpt); var output = parseResult.GetValue(outputOpt); var password = parseResult.GetValue(passwordOpt); var encodingName = parseResult.GetValue(encodingOpt)!; + var useDataSet = parseResult.GetValue(dataSetOpt) || fillMerged; // Expand comma-separated index tokens into a set of 1-based indices. var nameSet = new HashSet(sheetNames, StringComparer.Ordinal); @@ -80,31 +83,43 @@ static Command BuildExcelCommand() var openMs = sw.ElapsedMilliseconds; - var ds = reader.AsDataSet(new ExcelDataSetConfiguration + if (useDataSet) { - UseColumnDataType = false, - FilterSheet = (tableReader, sheetIndex) => + var ds = reader.AsDataSet(new ExcelDataSetConfiguration { - if (!hasFilter) - return true; + UseColumnDataType = false, + FilterSheet = (tableReader, sheetIndex) => + { + if (!hasFilter) + return true; - // sheetIndex is 0-based; expose as 1-based to the user. - return nameSet.Contains(tableReader.Name) || indexSet.Contains(sheetIndex + 1); - }, - ConfigureDataTable = _ => new ExcelDataTableConfiguration - { - UseHeaderRow = !noHeader, - FillMergedCellsValue = !noFillMerged, - } - }); + // sheetIndex is 0-based; expose as 1-based to the user. + return nameSet.Contains(tableReader.Name) || indexSet.Contains(sheetIndex + 1); + }, + ConfigureDataTable = _ => new ExcelDataTableConfiguration + { + UseHeaderRow = !noHeader, + FillMergedCellsValue = fillMerged, + } + }); - var readMs = sw.ElapsedMilliseconds - openMs; - var memAfter = Process.GetCurrentProcess().WorkingSet64; + var readMs = sw.ElapsedMilliseconds - openMs; + var memAfter = Process.GetCurrentProcess().WorkingSet64; - PrintStats(openMs, readMs, ds, memBefore, memAfter); + PrintStats(openMs, readMs, ds, memBefore, memAfter); - if (output != OutputFormat.None) - RenderSheets(ds, output); + if (output != OutputFormat.None) + RenderSheets(ds, output); + } + else + { + var sheets = ReadRaw(reader, noHeader, hasFilter, nameSet, indexSet, output); + + var readMs = sw.ElapsedMilliseconds - openMs; + var memAfter = Process.GetCurrentProcess().WorkingSet64; + + PrintRawStats(openMs, readMs, sheets, memBefore, memAfter); + } }); return cmd; @@ -121,6 +136,7 @@ static Command BuildCsvCommand() var separatorsOpt = new Option("--separators") { Description = "Separator candidates, e.g. \",;\" -- use \\t for TAB (default: , ; TAB | #)" }; var quoteCharOpt = new Option("--quote-char") { Description = "Quote character (default: \")" }; var escapeCharOpt = new Option("--escape-char") { Description = "Escape character for quoted fields (default: disabled)" }; + var dataSetOpt = new Option("--dataset") { Description = "Use AsDataSet extension (loads all data into a DataSet in memory)" }; var cmd = new Command("csv", "Read CSV files"); cmd.Arguments.Add(fileArg); @@ -131,6 +147,7 @@ static Command BuildCsvCommand() cmd.Options.Add(separatorsOpt); cmd.Options.Add(quoteCharOpt); cmd.Options.Add(escapeCharOpt); + cmd.Options.Add(dataSetOpt); cmd.SetAction(parseResult => { @@ -142,6 +159,7 @@ static Command BuildCsvCommand() var separatorsStr = parseResult.GetValue(separatorsOpt); var quoteCharStr = parseResult.GetValue(quoteCharOpt); var escapeCharStr = parseResult.GetValue(escapeCharOpt); + var useDataSet = parseResult.GetValue(dataSetOpt); var config = new ExcelReaderConfiguration { @@ -161,27 +179,125 @@ static Command BuildCsvCommand() var openMs = sw.ElapsedMilliseconds; - var ds = reader.AsDataSet(new ExcelDataSetConfiguration + if (useDataSet) { - UseColumnDataType = false, - ConfigureDataTable = _ => new ExcelDataTableConfiguration + var ds = reader.AsDataSet(new ExcelDataSetConfiguration { - UseHeaderRow = !noHeader, - } - }); + UseColumnDataType = false, + ConfigureDataTable = _ => new ExcelDataTableConfiguration + { + UseHeaderRow = !noHeader, + } + }); + + var readMs = sw.ElapsedMilliseconds - openMs; + var memAfter = Process.GetCurrentProcess().WorkingSet64; + + PrintStats(openMs, readMs, ds, memBefore, memAfter); - var readMs = sw.ElapsedMilliseconds - openMs; - var memAfter = Process.GetCurrentProcess().WorkingSet64; + if (output != OutputFormat.None) + RenderSheets(ds, output); + } + else + { + var sheets = ReadRaw(reader, noHeader, hasFilter: false, nameSet: [], indexSet: [], output); - PrintStats(openMs, readMs, ds, memBefore, memAfter); + var readMs = sw.ElapsedMilliseconds - openMs; + var memAfter = Process.GetCurrentProcess().WorkingSet64; - if (output != OutputFormat.None) - RenderSheets(ds, output); + PrintRawStats(openMs, readMs, sheets, memBefore, memAfter); + } }); return cmd; } +// ---- raw reader loop -------------------------------------------------------- +static List<(string Name, long Rows, int Cols)> ReadRaw( + IExcelDataReader reader, + bool noHeader, + bool hasFilter, + HashSet nameSet, + HashSet indexSet, + OutputFormat output) +{ + var sheets = new List<(string Name, long Rows, int Cols)>(); + bool firstOutput = true; + int sheetNumber = 0; + const int tableRowCap = 100; + + do + { + sheetNumber++; + if (hasFilter && !nameSet.Contains(reader.Name) && !indexSet.Contains(sheetNumber)) + continue; + + string sheetName = reader.Name; + long sheetRows = 0; + int sheetCols = 0; + string[]? headers = null; + List? tableBuffer = output == OutputFormat.Table ? [] : null; + + // Read header row if applicable. + if (!noHeader && reader.Read()) + { + headers = new string[reader.FieldCount]; + for (int i = 0; i < reader.FieldCount; i++) + headers[i] = reader.GetValue(i)?.ToString() ?? string.Empty; + + sheetCols = Math.Max(sheetCols, reader.FieldCount); + } + + // Emit header to separated output immediately. + if (output is OutputFormat.Csv or OutputFormat.Tsv) + { + char sep = output == OutputFormat.Csv ? ',' : '\t'; + if (!firstOutput) + Console.WriteLine(); + + if (sheets.Count > 0) + Console.WriteLine($"# Sheet: {sheetName}"); + + if (headers is not null) + WriteSeparatedRow(headers, sep); + } + + // Read data rows. + while (reader.Read()) + { + sheetRows++; + sheetCols = Math.Max(sheetCols, reader.FieldCount); + + var cells = new string[reader.FieldCount]; + for (int i = 0; i < reader.FieldCount; i++) + cells[i] = reader.GetValue(i)?.ToString() ?? string.Empty; + + if (output is OutputFormat.Csv or OutputFormat.Tsv) + WriteSeparatedRow(cells, output == OutputFormat.Csv ? ',' : '\t'); + else if (output == OutputFormat.Table && sheetRows <= tableRowCap) + tableBuffer!.Add(cells); + } + + // Render buffered table output. + if (output == OutputFormat.Table) + { + if (sheets.Count > 0) + AnsiConsole.MarkupLine($"[bold]{Markup.Escape(sheetName)}[/]"); + + if (sheetRows > tableRowCap) + Console.Error.WriteLine($"(showing first {tableRowCap:N0} of {sheetRows:N0} rows — use --output csv for full output)"); + + RenderRawTable(headers, tableBuffer!); + } + + sheets.Add((sheetName, sheetRows, sheetCols)); + firstOutput = false; + } + while (reader.NextResult()); + + return sheets; +} + // ---- helpers ---------------------------------------------------------------- static void PrintStats(long openMs, long readMs, DataSet ds, long memBefore, long memAfter) { @@ -195,8 +311,8 @@ static void PrintStats(long openMs, long readMs, DataSet ds, long memBefore, lon maxCols = dt.Columns.Count; } - long totalCells = totalRows * maxCols; - double cellsPerSec = readMs > 0 ? totalCells * 1000.0 / readMs : double.PositiveInfinity; + long totalMs = openMs + readMs; + string rateStr = BuildRateStr(totalRows * maxCols, totalMs); long memDeltaMb = (memAfter - memBefore) / (1024 * 1024); Console.Error.WriteLine($"Open: {openMs,6:N0} ms"); @@ -212,10 +328,49 @@ static void PrintStats(long openMs, long readMs, DataSet ds, long memBefore, lon Console.Error.WriteLine(")"); } - Console.Error.WriteLine($"Total: {openMs + readMs,6:N0} ms (~{cellsPerSec / 1_000_000:F1}M cells/sec)"); + Console.Error.WriteLine($"Total: {totalMs,6:N0} ms{rateStr}"); Console.Error.WriteLine($"Memory: {(memDeltaMb >= 0 ? "+" : string.Empty)}{memDeltaMb} MB"); } +static void PrintRawStats(long openMs, long readMs, List<(string Name, long Rows, int Cols)> sheets, long memBefore, long memAfter) +{ + long totalRows = sheets.Sum(s => s.Rows); + int maxCols = sheets.Count > 0 ? sheets.Max(s => s.Cols) : 0; + + long totalMs = openMs + readMs; + string rateStr = BuildRateStr(totalRows * maxCols, totalMs); + long memDeltaMb = (memAfter - memBefore) / (1024 * 1024); + + Console.Error.WriteLine($"Open: {openMs,6:N0} ms"); + Console.Error.Write($"Read: {readMs,6:N0} ms ({totalRows:N0} rows x {maxCols} cols"); + if (sheets.Count > 1) + { + Console.Error.WriteLine($" across {sheets.Count} sheets)"); + foreach (var (name, rows, cols) in sheets) + Console.Error.WriteLine($" {name}: {rows:N0} rows x {cols} cols"); + } + else + { + Console.Error.WriteLine(")"); + } + + Console.Error.WriteLine($"Total: {totalMs,6:N0} ms{rateStr}"); + Console.Error.WriteLine($"Memory: {(memDeltaMb >= 0 ? "+" : string.Empty)}{memDeltaMb} MB"); +} + +static string BuildRateStr(long totalCells, long totalMs) +{ + if (totalMs <= 0) + return string.Empty; + + double cellsPerSec = totalCells * 1000.0 / totalMs; + return cellsPerSec >= 1_000_000 + ? $" (~{cellsPerSec / 1_000_000:F1}M cells/sec)" + : cellsPerSec >= 1_000 + ? $" (~{cellsPerSec / 1_000:F1}K cells/sec)" + : $" (~{cellsPerSec:F0} cells/sec)"; +} + static void RenderSheets(DataSet ds, OutputFormat output) { bool multiSheet = ds.Tables.Count > 1; @@ -244,12 +399,43 @@ static void RenderSheets(DataSet ds, OutputFormat output) static void RenderSpectreTable(DataTable dt) { + const int tableRowCap = 100; var table = new Table(); foreach (DataColumn col in dt.Columns) table.AddColumn(Markup.Escape(col.ColumnName)); + int rendered = 0; foreach (DataRow row in dt.Rows) + { + if (rendered >= tableRowCap) + break; + table.AddRow(row.ItemArray.Select(v => Markup.Escape(v?.ToString() ?? string.Empty)).ToArray()); + rendered++; + } + + if (dt.Rows.Count > tableRowCap) + Console.Error.WriteLine($"(showing first {tableRowCap:N0} of {dt.Rows.Count:N0} rows — use --output csv for full output)"); + + AnsiConsole.Write(table); +} + +static void RenderRawTable(string[]? headers, List rows) +{ + var table = new Table(); + if (headers is not null) + { + foreach (var h in headers) + table.AddColumn(Markup.Escape(h)); + } + else if (rows.Count > 0) + { + for (int i = 0; i < rows[0].Length; i++) + table.AddColumn($"Col{i + 1}"); + } + + foreach (var row in rows) + table.AddRow(row.Select(v => Markup.Escape(v)).ToArray()); AnsiConsole.Write(table); } @@ -265,6 +451,16 @@ static string Escape(string s, char sep) => Console.WriteLine(string.Join(sep, row.ItemArray.Select(v => Escape(v?.ToString() ?? string.Empty, sep)))); } +static void WriteSeparatedRow(string[] cells, char sep) +{ + static string Escape(string s, char sep) => + s.Contains(sep) || s.Contains('"') || s.Contains('\n') + ? $"\"{s.Replace("\"", "\"\"")}\"" + : s; + + Console.WriteLine(string.Join(sep, cells.Select(v => Escape(v, sep)))); +} + #pragma warning disable SA1649 // File name should match first type name (top-level statements use implicit Program class) internal enum OutputFormat { @@ -272,4 +468,4 @@ internal enum OutputFormat Table, Csv, Tsv, -} \ No newline at end of file +} From f1dbd25a739d17d0b39235769e751e9f69e44082 Mon Sep 17 00:00:00 2001 From: Johan Appelgren Date: Wed, 27 May 2026 14:34:18 +0200 Subject: [PATCH 7/8] Fix table rendering for ragged rows (e.g. in single-pass mode) In single-pass mode, trailing empty cells may not be present in every row, leading to varying field counts. RenderRawTable now computes the true max column count across all buffered rows and pads shorter rows with empty strings to prevent Spectre.Console from throwing. --- src/ExcelDataReader.ConsoleSample/Program.cs | 23 +++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/ExcelDataReader.ConsoleSample/Program.cs b/src/ExcelDataReader.ConsoleSample/Program.cs index b70bf947..71757533 100644 --- a/src/ExcelDataReader.ConsoleSample/Program.cs +++ b/src/ExcelDataReader.ConsoleSample/Program.cs @@ -422,20 +422,37 @@ static void RenderSpectreTable(DataTable dt) static void RenderRawTable(string[]? headers, List rows) { + // Determine the actual maximum column count across headers and all rows. + // In single-pass mode rows may have fewer fields than the header (trailing + // empty cells are not stored), and later rows may have more columns than + // earlier ones. Pad everything to the widest row. + int colCount = headers?.Length ?? 0; + foreach (var row in rows) + colCount = Math.Max(colCount, row.Length); + var table = new Table(); if (headers is not null) { foreach (var h in headers) table.AddColumn(Markup.Escape(h)); + + for (int i = headers.Length; i < colCount; i++) + table.AddColumn($"Col{i + 1}"); } - else if (rows.Count > 0) + else { - for (int i = 0; i < rows[0].Length; i++) + for (int i = 0; i < colCount; i++) table.AddColumn($"Col{i + 1}"); } foreach (var row in rows) - table.AddRow(row.Select(v => Markup.Escape(v)).ToArray()); + { + var cells = new string[colCount]; + for (int i = 0; i < colCount; i++) + cells[i] = i < row.Length ? Markup.Escape(row[i]) : string.Empty; + + table.AddRow(cells); + } AnsiConsole.Write(table); } From 715546d7e5e5a93293ec2fb5a2a2227c2b88f6c6 Mon Sep 17 00:00:00 2001 From: Johan Appelgren Date: Wed, 27 May 2026 14:35:09 +0200 Subject: [PATCH 8/8] Fix CSV escaping: also quote fields containing bare carriage return --- src/ExcelDataReader.ConsoleSample/Program.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ExcelDataReader.ConsoleSample/Program.cs b/src/ExcelDataReader.ConsoleSample/Program.cs index 71757533..e1f7fff8 100644 --- a/src/ExcelDataReader.ConsoleSample/Program.cs +++ b/src/ExcelDataReader.ConsoleSample/Program.cs @@ -460,7 +460,7 @@ static void RenderRawTable(string[]? headers, List rows) static void WriteSeparated(DataTable dt, char sep) { static string Escape(string s, char sep) => - s.Contains(sep) || s.Contains('"') || s.Contains('\n') + s.Contains(sep) || s.Contains('"') || s.Contains('\n') || s.Contains('\r') ? $"\"{s.Replace("\"", "\"\"")}\"" : s; @@ -471,7 +471,7 @@ static string Escape(string s, char sep) => static void WriteSeparatedRow(string[] cells, char sep) { static string Escape(string s, char sep) => - s.Contains(sep) || s.Contains('"') || s.Contains('\n') + s.Contains(sep) || s.Contains('"') || s.Contains('\n') || s.Contains('\r') ? $"\"{s.Replace("\"", "\"\"")}\"" : s;