From 8379db36c2b03ed00fc368aabdedbca742d73b19 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Mon, 5 Oct 2026 15:21:18 -0300 Subject: [PATCH 01/28] feat: enhance support for various Arrow types and improve handling of empty rows in readers --- docs/guide/reading.md | 2 +- src/ExcelReader.Arrow/ArrowWriteExtensions.cs | 98 ++++++++++-- .../Reader/Csv/CsvStructuralScanner.cs | 13 +- .../Reader/Xls/XlsReader.Enumerator.cs | 33 ++++- src/ExcelReader.Core/Reader/Xls/XlsReader.cs | 1 + .../Reader/Xlsb/XlsbReader.Enumerator.cs | 78 +++------- src/ExcelReader.Core/Writer/Xls/BiffRecord.cs | 1 + .../Writer/Xls/BiffRecordWriter.cs | 7 + .../Writer/Xls/XlsSheetWriter.cs | 9 ++ .../Arrow/ArrowWriteExtensionsTests.cs | 140 +++++++++++++++++- .../Reader/Csv/CsvStructuralScannerTests.cs | 23 +++ .../Reader/EmptyRowParityTests.cs | 113 ++++++++++++++ .../Reader/Xlsb/XlsbEnumeratorTests.cs | 11 +- 13 files changed, 448 insertions(+), 81 deletions(-) create mode 100644 tests/ExcelReader.Tests/Reader/Csv/CsvStructuralScannerTests.cs create mode 100644 tests/ExcelReader.Tests/Reader/EmptyRowParityTests.cs diff --git a/docs/guide/reading.md b/docs/guide/reading.md index 83ad3c87..142c5d18 100644 --- a/docs/guide/reading.md +++ b/docs/guide/reading.md @@ -107,7 +107,7 @@ using XlsxWorkbookWriter workbook = XlsxWorkbookWriter.Create(File.Create("repor workbook.WriteRecordBatch(batch); // adds the sheet, writes the header + every row, ends the workbook ``` -Supports the same seven Arrow types `ToArrowRecordBatch` produces (string, int64, double, boolean, date32, time64, timestamp); any other type throws `NotSupportedException`. Pass `writeHeader: false` to skip the header row, or a `sheetName` (XLSX/XLSB/XLS only — CSV has no sheet name) to rename it from the `"Sheet1"` default. +Supports string, large string, string view, every integer width (signed and unsigned), half float, float, double, decimal128, decimal256, boolean, date32, date64, time32, time64, timestamp and null columns; any other type (binary, nested, dictionary) throws `NotSupportedException`. Pass `writeHeader: false` to skip the header row, or a `sheetName` (XLSX/XLSB/XLS only — CSV has no sheet name) to rename it from the `"Sheet1"` default. ## Open by auto-detecting the format diff --git a/src/ExcelReader.Arrow/ArrowWriteExtensions.cs b/src/ExcelReader.Arrow/ArrowWriteExtensions.cs index b43db774..a1860064 100644 --- a/src/ExcelReader.Arrow/ArrowWriteExtensions.cs +++ b/src/ExcelReader.Arrow/ArrowWriteExtensions.cs @@ -1,3 +1,4 @@ +using System.Globalization; using Apache.Arrow; using Apache.Arrow.Types; using ExcelReader.Core.Writer; @@ -24,13 +25,14 @@ public static class ArrowWriteExtensions /// /// A freshly created, not-yet-started workbook writer. /// - /// The batch to write. Only the seven Arrow types - /// produces (string, int64, double, boolean, date32, time64, timestamp) are supported. + /// The batch to write. Supported Arrow types: string, large string, string view, every integer + /// width (signed and unsigned), half float, float, double, decimal128, decimal256, boolean, + /// date32, date64, time32, time64, timestamp and null. /// /// The sheet's name. /// Whether to write a header row of column names first. /// or is . - /// A column's Arrow type is not one of the seven supported types. + /// A column's Arrow type is not one of the supported types. public static void WriteRecordBatch(this XlsxWorkbookWriter workbook, RecordBatch batch, string sheetName = "Sheet1", bool writeHeader = true) { WriteRecordBatchCore(workbook, batch, sheetName, writeHeader); @@ -38,7 +40,7 @@ public static void WriteRecordBatch(this XlsxWorkbookWriter workbook, RecordBatc /// Same as , for an XLSB workbook. /// or is . - /// A column's Arrow type is not one of the seven supported types. + /// A column's Arrow type is not one of the supported types. public static void WriteRecordBatch(this XlsbWorkbookWriter workbook, RecordBatch batch, string sheetName = "Sheet1", bool writeHeader = true) { WriteRecordBatchCore(workbook, batch, sheetName, writeHeader); @@ -46,7 +48,7 @@ public static void WriteRecordBatch(this XlsbWorkbookWriter workbook, RecordBatc /// Same as , for a legacy XLS workbook. /// or is . - /// A column's Arrow type is not one of the seven supported types. + /// A column's Arrow type is not one of the supported types. public static void WriteRecordBatch(this XlsWorkbookWriter workbook, RecordBatch batch, string sheetName = "Sheet1", bool writeHeader = true) { WriteRecordBatchCore(workbook, batch, sheetName, writeHeader); @@ -57,7 +59,7 @@ public static void WriteRecordBatch(this XlsWorkbookWriter workbook, RecordBatch /// in a single call. /// /// or is . - /// A column's Arrow type is not one of the seven supported types. + /// A column's Arrow type is not one of the supported types. public static void WriteRecordBatch(this CsvWorkbookWriter workbook, RecordBatch batch, bool writeHeader = true) { WriteRecordBatchCore(workbook, batch, sheetName: "Sheet1", writeHeader); @@ -166,8 +168,10 @@ private static void ApplyTemporalStyles(IWorkbookWriter wo switch (fields[i].DataType.TypeId) { case ArrowTypeId.Date32: + case ArrowTypeId.Date64: sheet.SetColumnStyle(i, BuiltinDateStyleId); break; + case ArrowTypeId.Time32: case ArrowTypeId.Time64: timeStyle = timeStyle < 0 ? workbook.AddStyle(new CellStyle { NumberFormat = "hh:mm:ss" }) : timeStyle; sheet.SetColumnStyle(i, timeStyle); @@ -187,13 +191,55 @@ private static void WriteCell(IRowWriter row, IArrowArray array, ArrowTypeId typ switch (typeId) { case ArrowTypeId.String: - var strings = (StringArray)array; - if (strings.IsNull(index)) - { - row.Write((string?)null); - return; - } - row.WriteUtf8(strings.GetBytes(index)); + WriteText(row, ((StringArray)array).GetBytes(index, out bool isNull), isNull); + return; + case ArrowTypeId.LargeString: + WriteText(row, ((LargeStringArray)array).GetBytes(index, out bool isLargeNull), isLargeNull); + return; + case ArrowTypeId.StringView: + WriteText(row, ((StringViewArray)array).GetBytes(index, out bool isViewNull), isViewNull); + return; + case ArrowTypeId.Null: + row.Write((string?)null); + return; + case ArrowTypeId.Int8: + row.Write(((Int8Array)array).GetValue(index)); + return; + case ArrowTypeId.Int16: + row.Write(((Int16Array)array).GetValue(index)); + return; + case ArrowTypeId.Int32: + row.Write(((Int32Array)array).GetValue(index)); + return; + case ArrowTypeId.UInt8: + row.Write(((UInt8Array)array).GetValue(index)); + return; + case ArrowTypeId.UInt16: + row.Write(((UInt16Array)array).GetValue(index)); + return; + case ArrowTypeId.UInt32: + row.Write(((UInt32Array)array).GetValue(index)); + return; + case ArrowTypeId.UInt64: + row.Write(((UInt64Array)array).GetValue(index)); + return; + case ArrowTypeId.HalfFloat: + row.Write(ToShortestDouble(((HalfFloatArray)array).GetValue(index))); + return; + case ArrowTypeId.Float: + row.Write(ToShortestDouble(((FloatArray)array).GetValue(index))); + return; + case ArrowTypeId.Decimal128: + row.Write(((Decimal128Array)array).GetValue(index)); + return; + case ArrowTypeId.Decimal256: + row.Write(((Decimal256Array)array).GetValue(index)); + return; + case ArrowTypeId.Date64: + row.Write(((Date64Array)array).GetDateOnly(index)); + return; + case ArrowTypeId.Time32: + row.Write(((Time32Array)array).GetTime(index)); return; case ArrowTypeId.Int64: row.Write(((Int64Array)array).GetValue(index)); @@ -217,5 +263,31 @@ private static void WriteCell(IRowWriter row, IArrowArray array, ArrowTypeId typ throw new NotSupportedException($"Arrow type {typeId} is not supported for writing."); } } + + /// + /// Widens through the value's shortest decimal text, so 0.1f lands as 0.1 in every + /// format instead of the binary writers storing 0.10000000149011612. + /// + private static double? ToShortestDouble(T? value) + where T : struct, IUtf8SpanFormattable + { + if (value is not T narrow) + { + return null; + } + Span text = stackalloc byte[32]; + narrow.TryFormat(text, out int written, default, CultureInfo.InvariantCulture); + return double.Parse(text[..written], NumberStyles.Float, CultureInfo.InvariantCulture); + } + + private static void WriteText(IRowWriter row, ReadOnlySpan utf8, bool isNull) + { + if (isNull) + { + row.Write((string?)null); + return; + } + row.WriteUtf8(utf8); + } } } diff --git a/src/ExcelReader.Core/Reader/Csv/CsvStructuralScanner.cs b/src/ExcelReader.Core/Reader/Csv/CsvStructuralScanner.cs index 4fd09299..593a6af9 100644 --- a/src/ExcelReader.Core/Reader/Csv/CsvStructuralScanner.cs +++ b/src/ExcelReader.Core/Reader/Csv/CsvStructuralScanner.cs @@ -3,6 +3,7 @@ using System.Runtime.InteropServices; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.X86; +using ArmAes = System.Runtime.Intrinsics.Arm.Aes; namespace ExcelReader.Core.Reader.Csv { @@ -258,14 +259,22 @@ private void Classify(ulong quotes, ulong delims, ulong crs, ulong lfs, ulong va _blocked = true; } - // ponytail: CLMUL on x86 only; ARM takes the shift ladder, add Aes.PolynomialMultiplyWideningLower if ARM throughput matters. [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static ulong PrefixXor(ulong bits) + internal static ulong PrefixXor(ulong bits) { if (Pclmulqdq.IsSupported) { return Pclmulqdq.CarrylessMultiply(Vector128.CreateScalar(bits), Vector128.AllBitsSet, 0).ToScalar(); } + if (ArmAes.IsSupported) + { + return ArmAes.PolynomialMultiplyWideningLower(Vector64.CreateScalar(bits), Vector64.AllBitsSet).ToScalar(); + } + return PrefixXorSoftware(bits); + } + + internal static ulong PrefixXorSoftware(ulong bits) + { bits ^= bits << 1; bits ^= bits << 2; bits ^= bits << 4; diff --git a/src/ExcelReader.Core/Reader/Xls/XlsReader.Enumerator.cs b/src/ExcelReader.Core/Reader/Xls/XlsReader.Enumerator.cs index deee8b98..6328efa3 100644 --- a/src/ExcelReader.Core/Reader/Xls/XlsReader.Enumerator.cs +++ b/src/ExcelReader.Core/Reader/Xls/XlsReader.Enumerator.cs @@ -17,8 +17,11 @@ public sealed class Enumerator : IExcelRowEnumerator private readonly BiffCursor _cursor; private readonly CellAccumulator _acc; private readonly Utf8StringCache? _contentCache; + private readonly Queue _declaredRows = new(); private bool _ended; + private bool _emptyRow; private int _row; + private int _rowFloor = -1; internal Enumerator(XlsReader reader, int sheetOffset, CancellationToken ct = default) { @@ -72,7 +75,8 @@ private bool MoveNextCore() return true; } } - return false; + ResetRow(); + return _declaredRows.TryDequeue(out _); } private bool ReadRecord(BiffCursor cursor, long recordStart, int id, ReadOnlySpan data) @@ -96,6 +100,9 @@ private bool ReadRecord(BiffCursor cursor, long recordStart, int id, ReadOnlySpa } switch (id) { + case Rec.Row: + DeclareRow(ReadU16(data, 0)); + return true; case Rec.Label: if (!AdvanceRow(cursor, recordStart, ReadU16(data, 0))) { return false; } ParseLabel(data); @@ -144,11 +151,32 @@ private bool ReadRecord(BiffCursor cursor, long recordStart, int id, ReadOnlySpa } } + /// Queues a ROW record's row so it is yielded, empty, if no cell record claims it. + private void DeclareRow(int row) + { + if (row > _rowFloor) + { + _rowFloor = row; + _declaredRows.Enqueue(row); + } + } + private bool AdvanceRow(BiffCursor cursor, long recordStart, int row) { if (_row < 0) { + if (_declaredRows.TryPeek(out int declared) && declared <= row) + { + _declaredRows.Dequeue(); + if (declared < row) + { + cursor.Position = recordStart; + _emptyRow = true; + return false; + } + } _row = row; + _rowFloor = Math.Max(_rowFloor, row); return true; } if (row != _row) @@ -162,13 +190,14 @@ private bool AdvanceRow(BiffCursor cursor, long recordStart, int row) private bool FinishRow() { _acc.SortByColumn(); - return _acc.Count > 0; + return _row >= 0 || _emptyRow; } private void ResetRow() { _acc.Reset(); _row = -1; + _emptyRow = false; } private void ParseLabel(ReadOnlySpan data) diff --git a/src/ExcelReader.Core/Reader/Xls/XlsReader.cs b/src/ExcelReader.Core/Reader/Xls/XlsReader.cs index 8ae7e95d..2d09e80d 100644 --- a/src/ExcelReader.Core/Reader/Xls/XlsReader.cs +++ b/src/ExcelReader.Core/Reader/Xls/XlsReader.cs @@ -428,6 +428,7 @@ private static class Rec internal const int MulRk = 0x00BD; internal const int BoolErr = 0x0205; internal const int Formula = 0x0006; + internal const int Row = 0x0208; internal const int Blank = 0x0201; internal const int MulBlank = 0x00BE; internal const int StringRec = 0x0207; diff --git a/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Enumerator.cs b/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Enumerator.cs index 614b2e52..5cc77df6 100644 --- a/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Enumerator.cs +++ b/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Enumerator.cs @@ -76,36 +76,25 @@ public ValueTask MoveNextAsync() { return new ValueTask(false); } - while (true) + ResetRow(); + if (!_pendingRowHdr) { - ResetRow(); - if (!_pendingRowHdr) - { - int seek = SeekRowHdrFromBuffer(); - if (seek == 0) - { - return new ValueTask(false); - } - if (seek == 2) - { - return MoveNextRowAsync(seekDone: false); - } - } - _pendingRowHdr = false; - int collect = CollectCellsFromBuffer(); - if (collect == 2) - { - return MoveNextRowAsync(seekDone: true); - } - if (_acc.Count > 0) + int seek = SeekRowHdrFromBuffer(); + if (seek == 0) { - return new ValueTask(true); + return new ValueTask(false); } - if (!_pendingRowHdr) + if (seek == 2) { - return new ValueTask(false); + return MoveNextRowAsync(seekDone: false); } } + _pendingRowHdr = false; + if (CollectCellsFromBuffer() == 2) + { + return MoveNextRowAsync(seekDone: true); + } + return new ValueTask(true); } private async ValueTask MoveNextRowAsync(bool seekDone) @@ -116,24 +105,7 @@ private async ValueTask MoveNextRowAsync(bool seekDone) } _pendingRowHdr = false; await CollectCellsAsync().ConfigureAwait(false); - while (true) - { - if (_acc.Count > 0) - { - return true; - } - if (!_pendingRowHdr) - { - return false; - } - ResetRow(); - if (!await SeekRowHdrAsync().ConfigureAwait(false)) - { - return false; - } - _pendingRowHdr = false; - await CollectCellsAsync().ConfigureAwait(false); - } + return true; } private bool MoveNextCore() @@ -142,24 +114,14 @@ private bool MoveNextCore() { return false; } - while (true) + ResetRow(); + if (!_pendingRowHdr && !SkipToRowHdr()) { - ResetRow(); - if (!_pendingRowHdr && !SkipToRowHdr()) - { - return false; - } - _pendingRowHdr = false; - CollectCells(); - if (_acc.Count > 0) - { - return true; - } - if (!_pendingRowHdr) - { - return false; - } + return false; } + _pendingRowHdr = false; + CollectCells(); + return true; } private bool SkipToRowHdr() diff --git a/src/ExcelReader.Core/Writer/Xls/BiffRecord.cs b/src/ExcelReader.Core/Writer/Xls/BiffRecord.cs index 47e78a12..53f15481 100644 --- a/src/ExcelReader.Core/Writer/Xls/BiffRecord.cs +++ b/src/ExcelReader.Core/Writer/Xls/BiffRecord.cs @@ -27,6 +27,7 @@ internal static class BiffRecord internal const int BookBool = 0x00DA; internal const int Dimension = 0x0200; internal const int Window2 = 0x023E; + internal const int Blank = 0x0201; internal const int Number = 0x0203; internal const int Label = 0x0204; internal const int BoolErr = 0x0205; diff --git a/src/ExcelReader.Core/Writer/Xls/BiffRecordWriter.cs b/src/ExcelReader.Core/Writer/Xls/BiffRecordWriter.cs index 053c54c9..1b205a00 100644 --- a/src/ExcelReader.Core/Writer/Xls/BiffRecordWriter.cs +++ b/src/ExcelReader.Core/Writer/Xls/BiffRecordWriter.cs @@ -218,6 +218,13 @@ internal static void WriteWindow2(BiffBuffer buffer) buffer.EndRecord(len); } + internal static void WriteBlank(BiffBuffer buffer, int row, int col, int xf) + { + int len = buffer.BeginRecord(BiffRecord.Blank); + WriteCellHeader(buffer, row, col, xf); + buffer.EndRecord(len); + } + internal static void WriteNumber(BiffBuffer buffer, int row, int col, int xf, double value) { CellValueGuards.ThrowIfNonFinite(value, nameof(value)); diff --git a/src/ExcelReader.Core/Writer/Xls/XlsSheetWriter.cs b/src/ExcelReader.Core/Writer/Xls/XlsSheetWriter.cs index 7bb20635..86517f61 100644 --- a/src/ExcelReader.Core/Writer/Xls/XlsSheetWriter.cs +++ b/src/ExcelReader.Core/Writer/Xls/XlsSheetWriter.cs @@ -31,6 +31,7 @@ public sealed class XlsSheetWriter : ISheetWriter private int _rowNumber = -1; private WriterState _state = WriterState.Created; private bool _rowActive; + private bool _rowHasCells; private Dictionary? _columnStyles; private Dictionary? _columnWidths; private int _activeRowStyle; @@ -158,13 +159,20 @@ public XlsRowWriter StartRow(int styleId) } _activeRowStyle = styleId; _rowActive = true; + _rowHasCells = false; _rowWriter ??= new XlsRowWriter(this, _rowNumber); _rowWriter.Reset(_rowNumber); return _rowWriter; } + /// A row left without cells gets one blank cell, since BIFF8 has no other trace of it for a reader to find. internal void NotifyRowEnded() { + if (_rowActive && !_rowHasCells) + { + BiffRecordWriter.WriteBlank(_cells, _rowNumber, 0, XlsGlobals.GeneralXf); + Track(_rowNumber, 0); + } _rowActive = false; } @@ -209,6 +217,7 @@ internal void ReleaseBuffer() private void Track(int row, int col) { + _rowHasCells = true; if (row > _maxRow) { _maxRow = row; } if (col > _maxCol) { _maxCol = col; } } diff --git a/tests/ExcelReader.Tests/Arrow/ArrowWriteExtensionsTests.cs b/tests/ExcelReader.Tests/Arrow/ArrowWriteExtensionsTests.cs index d52c5d4f..084d49f1 100644 --- a/tests/ExcelReader.Tests/Arrow/ArrowWriteExtensionsTests.cs +++ b/tests/ExcelReader.Tests/Arrow/ArrowWriteExtensionsTests.cs @@ -214,13 +214,147 @@ public void WriteRecordBatch_WithHeaderFalse_OmitsTheHeaderRow() Assert.False(e.MoveNext()); } + private static RecordBatch BuildWideBatch() + { + var decimalType = new Decimal128Type(10, 2); + var secondTime = new Time32Type(TimeUnit.Second); + + var i8 = new Int8Array.Builder(); + i8.Append(-8); + i8.Append(7); + var i16 = new Int16Array.Builder(); + i16.Append(-16); + i16.AppendNull(); + var i32 = new Int32Array.Builder(); + i32.Append(-32); + i32.AppendNull(); + var u8 = new UInt8Array.Builder(); + u8.Append(8); + u8.AppendNull(); + var u16 = new UInt16Array.Builder(); + u16.Append(16); + u16.AppendNull(); + var u32 = new UInt32Array.Builder(); + u32.Append(32); + u32.AppendNull(); + var u64 = new UInt64Array.Builder(); + u64.Append(64); + u64.AppendNull(); + var half = new HalfFloatArray.Builder(); + half.Append((Half)0.5f); + half.AppendNull(); + var single = new FloatArray.Builder(); + single.Append(0.1f); + single.AppendNull(); + var dec = new Decimal128Array.Builder(decimalType); + dec.Append(12.34m); + dec.AppendNull(); + var view = new StringViewArray.Builder(); + view.Append("alice"); + view.AppendNull(); + var date = new Date64Array.Builder(); + date.Append(new DateOnly(2024, 1, 15)); + date.AppendNull(); + var time = new Time32Array.Builder(secondTime); + time.Append(new TimeOnly(13, 30, 0)); + time.AppendNull(); + + Schema schema = new Schema.Builder() + .Field(new Field("i8", Int8Type.Default, nullable: true)) + .Field(new Field("i16", Int16Type.Default, nullable: true)) + .Field(new Field("i32", Int32Type.Default, nullable: true)) + .Field(new Field("u8", UInt8Type.Default, nullable: true)) + .Field(new Field("u16", UInt16Type.Default, nullable: true)) + .Field(new Field("u32", UInt32Type.Default, nullable: true)) + .Field(new Field("u64", UInt64Type.Default, nullable: true)) + .Field(new Field("half", HalfFloatType.Default, nullable: true)) + .Field(new Field("single", FloatType.Default, nullable: true)) + .Field(new Field("dec", decimalType, nullable: true)) + .Field(new Field("view", StringViewType.Default, nullable: true)) + .Field(new Field("date", Date64Type.Default, nullable: true)) + .Field(new Field("time", secondTime, nullable: true)) + .Field(new Field("nothing", NullType.Default, nullable: true)) + .Build(); + + return new RecordBatch( + schema, + [ + i8.Build(), i16.Build(), i32.Build(), u8.Build(), u16.Build(), u32.Build(), u64.Build(), + half.Build(), single.Build(), dec.Build(), view.Build(), date.Build(), time.Build(), new NullArray(2), + ], + 2); + } + + private static void AssertWideRoundTrips(IExcelRowReader reader) + { + ExcelColumnType[] types = + [ + ExcelColumnType.Int64Column, ExcelColumnType.Int64Column, ExcelColumnType.Int64Column, + ExcelColumnType.Int64Column, ExcelColumnType.Int64Column, ExcelColumnType.Int64Column, + ExcelColumnType.Int64Column, ExcelColumnType.Float64Column, ExcelColumnType.Float64Column, + ExcelColumnType.Float64Column, ExcelColumnType.StringColumn, ExcelColumnType.DateColumn, + ExcelColumnType.TimeColumn, ExcelColumnType.StringColumn, + ]; + var schema = new ExcelColumnSchema[types.Length]; + for (int i = 0; i < types.Length; i++) + { + schema[i] = new() { Index = i, Name = "c" + i, Type = types[i], IsNullable = true }; + } + RecordBatch batch = reader.ToArrowRecordBatch(schema); + + Assert.Equal(2, batch.Length); + long[] integers = [-8, -16, -32, 8, 16, 32, 64]; + for (int i = 0; i < integers.Length; i++) + { + Assert.Equal(integers[i], Assert.IsType(batch.Column(i)).GetValue(0)); + } + Assert.Equal(0.5, Assert.IsType(batch.Column(7)).GetValue(0)); + Assert.Equal(0.1, Assert.IsType(batch.Column(8)).GetValue(0)); + Assert.Equal(12.34, Assert.IsType(batch.Column(9)).GetValue(0)); + Assert.Equal("alice", Assert.IsType(batch.Column(10)).GetString(0)); + Assert.Equal(new DateOnly(2024, 1, 15), Assert.IsType(batch.Column(11)).GetDateOnly(0)); + Assert.Equal(new TimeOnly(13, 30, 0), Assert.IsType(batch.Column(12)).GetTime(0)); + Assert.Equal(7L, ((Int64Array)batch.Column(0)).GetValue(1)); + for (int i = 1; i < types.Length; i++) + { + Assert.True(batch.Column(i).IsNull(1), "column " + i); + } + Assert.True(batch.Column(13).IsNull(0)); + } + + [Fact] + public void WriteRecordBatch_Xlsx_RoundTrips_Widened_Types() + { + using var ms = new MemoryStream(); + using (XlsxWorkbookWriter workbook = XlsxWorkbookWriter.Create(ms, leaveOpen: true)) + { + workbook.WriteRecordBatch(BuildWideBatch()); + } + + using XlsxReader reader = Excel.FromXlsx(ms.ToArray()); + AssertWideRoundTrips(reader); + } + + [Fact] + public void WriteRecordBatch_Xlsb_RoundTrips_Widened_Types() + { + using var ms = new MemoryStream(); + using (XlsbWorkbookWriter workbook = XlsbWorkbookWriter.Create(ms, leaveOpen: true)) + { + workbook.WriteRecordBatch(BuildWideBatch()); + } + + using var reader = Excel.FromXlsb(ms.ToArray()); + AssertWideRoundTrips(reader); + } + [Fact] public void WriteRecordBatch_UnsupportedArrowType_Throws() { - var col = new FloatArray.Builder(); - col.Append(1.5f); + var col = new BinaryArray.Builder(); + col.Append(new byte[] { 1 }.AsSpan()); Schema schema = new Schema.Builder() - .Field(new Field("f", FloatType.Default, nullable: false)) + .Field(new Field("f", BinaryType.Default, nullable: false)) .Build(); var batch = new RecordBatch(schema, [col.Build()], 1); diff --git a/tests/ExcelReader.Tests/Reader/Csv/CsvStructuralScannerTests.cs b/tests/ExcelReader.Tests/Reader/Csv/CsvStructuralScannerTests.cs new file mode 100644 index 00000000..a485a550 --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Csv/CsvStructuralScannerTests.cs @@ -0,0 +1,23 @@ +using ExcelReader.Core.Reader.Csv; + +namespace ExcelReader.Tests.Reader.Csv +{ + public sealed class CsvStructuralScannerTests + { + [Fact] + public void PrefixXor_Matches_The_Software_Ladder() + { + ulong[] edges = [0, 1, 1UL << 63, ulong.MaxValue, 0x5555_5555_5555_5555, 0x8000_0000_0000_0001]; + foreach (ulong edge in edges) + { + Assert.Equal(CsvStructuralScanner.PrefixXorSoftware(edge), CsvStructuralScanner.PrefixXor(edge)); + } + ulong bits = 1; + for (int i = 0; i < 10_000; i++) + { + bits = (bits * 6364136223846793005UL) + 1442695040888963407UL; + Assert.Equal(CsvStructuralScanner.PrefixXorSoftware(bits), CsvStructuralScanner.PrefixXor(bits)); + } + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/EmptyRowParityTests.cs b/tests/ExcelReader.Tests/Reader/EmptyRowParityTests.cs new file mode 100644 index 00000000..4f1a172d --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/EmptyRowParityTests.cs @@ -0,0 +1,113 @@ +using ExcelReader.Core.Reader; +using ExcelReader.Core.Writer; +using ExcelReader.Core.Writer.Xls; +using ExcelReader.Core.Writer.Xlsb; +using ExcelReader.Core.Writer.Xlsx; +using ExcelReader.Tests.Reader.Xls; + +namespace ExcelReader.Tests.Reader +{ + public sealed class EmptyRowParityTests + { + private const int RowRecord = 0x0208; + private const int LabelRecord = 0x0204; + + private static void WriteSparseSheet(IWorkbookWriter workbook) + where TSheet : ISheetWriter + where TRow : IRowWriter + { + TSheet sheet = workbook.AddSheet("s"); + string?[] values = ["a", null, "b", null]; + foreach (string? value in values) + { + using TRow row = sheet.StartRow(); + row.Write(value); + row.Write((long?)null); + } + sheet.End(); + sheet.Dispose(); + workbook.End(); + } + + private static List FirstCells(TEnumerator e) + where TEnumerator : IExcelRowEnumerator + { + using TEnumerator owned = e; + var cells = new List(); + while (owned.MoveNext()) + { + cells.Add(owned.Current.ColumnCount == 0 ? null : owned.Current[0].GetString()); + } + return cells; + } + + [Fact] + public void Xlsx_Yields_Rows_Written_Without_Cells() + { + using var ms = new MemoryStream(); + using (XlsxWorkbookWriter workbook = XlsxWorkbookWriter.Create(ms, leaveOpen: true)) + { + WriteSparseSheet(workbook); + } + + using var reader = Excel.FromXlsx(ms.ToArray()); + Assert.Equal(["a", null, "b", null], FirstCells(reader.GetEnumerator())); + } + + [Fact] + public void Xlsb_Yields_Rows_Written_Without_Cells() + { + using var ms = new MemoryStream(); + using (XlsbWorkbookWriter workbook = XlsbWorkbookWriter.Create(ms, leaveOpen: true)) + { + WriteSparseSheet(workbook); + } + + using var reader = Excel.FromXlsb(ms.ToArray()); + Assert.Equal(["a", null, "b", null], FirstCells(reader.GetEnumerator())); + } + + [Fact] + public void Xls_Yields_Rows_Written_Without_Cells() + { + using var ms = new MemoryStream(); + using (XlsWorkbookWriter workbook = XlsWorkbookWriter.Create(ms, leaveOpen: true)) + { + WriteSparseSheet(workbook); + } + + using var reader = Excel.FromXls(ms.ToArray()); + Assert.Equal(["a", null, "b", null], FirstCells(reader.GetEnumerator())); + } + + [Fact] + public void Xls_Yields_A_Row_Record_Without_Cells_In_Row_Order() + { + using var ms = XlsWorkbookBuilder.BuildRawSheet( + includeEof: true, + (RowRecord, XlsWorkbookBuilder.RawRowOnly(0)), + (RowRecord, XlsWorkbookBuilder.RawRowOnly(1)), + (RowRecord, XlsWorkbookBuilder.RawRowOnly(2)), + (RowRecord, XlsWorkbookBuilder.RawRowOnly(3)), + (LabelRecord, XlsWorkbookBuilder.RawLabel(0, 0, "a")), + (LabelRecord, XlsWorkbookBuilder.RawLabel(2, 0, "b"))); + + using var reader = Excel.FromXls(ms); + Assert.Equal(["a", null, "b", null], FirstCells(reader.GetEnumerator())); + } + + [Fact] + public void Xls_Does_Not_Repeat_A_Row_Whose_Row_Record_Follows_Its_Cells() + { + using var ms = XlsWorkbookBuilder.BuildRawSheet( + includeEof: true, + (LabelRecord, XlsWorkbookBuilder.RawLabel(0, 0, "a")), + (LabelRecord, XlsWorkbookBuilder.RawLabel(1, 0, "b")), + (RowRecord, XlsWorkbookBuilder.RawRowOnly(0)), + (RowRecord, XlsWorkbookBuilder.RawRowOnly(1))); + + using var reader = Excel.FromXls(ms); + Assert.Equal(["a", "b"], FirstCells(reader.GetEnumerator())); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Xlsb/XlsbEnumeratorTests.cs b/tests/ExcelReader.Tests/Reader/Xlsb/XlsbEnumeratorTests.cs index b37b39bb..e1b51ebf 100644 --- a/tests/ExcelReader.Tests/Reader/Xlsb/XlsbEnumeratorTests.cs +++ b/tests/ExcelReader.Tests/Reader/Xlsb/XlsbEnumeratorTests.cs @@ -42,10 +42,12 @@ public void EmptySheetYieldsNoRows() } [Fact] - public void EmptyRowIsSkipped() + public void EmptyRowIsYielded() { byte[] sheet = [.. B.Record(Brt.RowHdr), .. B.Record(Brt.EndSheetData)]; using var e = Open(BlankReader(), sheet); + Assert.True(e.MoveNext()); + Assert.Equal(0, e.Current.ColumnCount); Assert.False(e.MoveNext()); } @@ -60,6 +62,8 @@ .. B.Record(Brt.RowHdr), .. B.Record(Brt.CellBool, B.CellBool(0, 0, true)), ]; using var e = Open(BlankReader(), sheet); + Assert.True(e.MoveNext()); + Assert.Equal(0, e.Current.ColumnCount); Assert.False(e.MoveNext()); } @@ -236,7 +240,7 @@ .. B.Record(Brt.EndSheetData), } [Fact] - public void EmptyRowsBetweenDataRowsAreSkipped() + public void EmptyRowsBetweenDataRowsAreYielded() { byte[] sheet = [ @@ -251,6 +255,9 @@ .. B.Record(Brt.CellBool, B.CellBool(0, 0, false)), Assert.True(e.MoveNext()); Assert.Equal("1", e.Current[0].GetString()); + Assert.True(e.MoveNext()); + Assert.Equal(0, e.Current.ColumnCount); + Assert.True(e.MoveNext()); Assert.Equal("0", e.Current[0].GetString()); From 74050b33c7a6ccf4998cbae788adf7a0bccf4511 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Tue, 6 Oct 2026 00:54:03 -0300 Subject: [PATCH 02/28] feat(core): add ByteSource, a positional view over memory, files and seekable streams Co-Authored-By: Claude Opus 5.5 --- .../Reader/Sources/ByteSource.cs | 137 +++++++++++++ .../Reader/Sources/FileByteSource.cs | 34 ++++ .../Reader/Sources/MemoryByteSource.cs | 43 ++++ .../Reader/Sources/StreamByteSource.cs | 67 ++++++ .../Reader/Sources/ByteSourceTests.cs | 192 ++++++++++++++++++ tests/ExcelReader.Tests/TestUtils.cs | 73 +++++++ 6 files changed, 546 insertions(+) create mode 100644 src/ExcelReader.Core/Reader/Sources/ByteSource.cs create mode 100644 src/ExcelReader.Core/Reader/Sources/FileByteSource.cs create mode 100644 src/ExcelReader.Core/Reader/Sources/MemoryByteSource.cs create mode 100644 src/ExcelReader.Core/Reader/Sources/StreamByteSource.cs create mode 100644 tests/ExcelReader.Tests/Reader/Sources/ByteSourceTests.cs diff --git a/src/ExcelReader.Core/Reader/Sources/ByteSource.cs b/src/ExcelReader.Core/Reader/Sources/ByteSource.cs new file mode 100644 index 00000000..7005cd6b --- /dev/null +++ b/src/ExcelReader.Core/Reader/Sources/ByteSource.cs @@ -0,0 +1,137 @@ +using Microsoft.Win32.SafeHandles; + +namespace ExcelReader.Core.Reader.Sources +{ + /// + /// Random-access bytes of one workbook file. Every read names its own offset, so readers of the + /// same source share no cursor. + /// + internal abstract class ByteSource : IDisposable + { + internal abstract long Length { get; } + + internal abstract int Read(long offset, Span destination); + + internal abstract ValueTask ReadAsync(long offset, Memory destination, CancellationToken ct); + + internal virtual bool TryGetMemory(out ReadOnlyMemory memory) + { + memory = default; + return false; + } + + internal void ReadExactly(long offset, Span destination) + { + int filled = 0; + while (filled < destination.Length) + { + int read = Read(offset + filled, destination[filled..]); + if (read <= 0) + { + throw new EndOfStreamException(); + } + filled += read; + } + } + + internal async ValueTask ReadExactlyAsync(long offset, Memory destination, CancellationToken ct) + { + int filled = 0; + while (filled < destination.Length) + { + int read = await ReadAsync(offset + filled, destination[filled..], ct).ConfigureAwait(false); + if (read <= 0) + { + throw new EndOfStreamException(); + } + filled += read; + } + } + + public abstract void Dispose(); + + internal static ByteSource FromMemory(ReadOnlyMemory data) + { + return new MemoryByteSource(data, owner: null); + } + + internal static ByteSource OpenFile(string path, bool asynchronous) + { + SafeFileHandle handle = File.OpenHandle(path, FileMode.Open, FileAccess.Read, FileShare.Read, + asynchronous ? FileOptions.Asynchronous : FileOptions.None); + try + { + return new FileByteSource(handle, RandomAccess.GetLength(handle), owner: handle); + } + catch + { + handle.Dispose(); + throw; + } + } + + internal static ByteSource FromStream(Stream stream, bool leaveOpen) + { + if (TryWrapSeekable(stream, leaveOpen, out ByteSource? source)) + { + return source; + } + try + { + using MemoryStream copy = new(); + stream.CopyTo(copy); + return new MemoryByteSource(copy.GetBuffer().AsMemory(0, (int)copy.Length), owner: null); + } + finally + { + if (!leaveOpen) + { + stream.Dispose(); + } + } + } + + internal static async ValueTask FromStreamAsync(Stream stream, bool leaveOpen, CancellationToken ct) + { + if (TryWrapSeekable(stream, leaveOpen, out ByteSource? source)) + { + return source; + } + try + { + using MemoryStream copy = new(); + await stream.CopyToAsync(copy, ct).ConfigureAwait(false); + return new MemoryByteSource(copy.GetBuffer().AsMemory(0, (int)copy.Length), owner: null); + } + finally + { + if (!leaveOpen) + { + await stream.DisposeAsync().ConfigureAwait(false); + } + } + } + + private static bool TryWrapSeekable(Stream stream, bool leaveOpen, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out ByteSource? source) + { + IDisposable? owner = leaveOpen ? null : stream; + if (stream is MemoryStream memory && memory.TryGetBuffer(out ArraySegment segment)) + { + source = new MemoryByteSource(segment, owner); + return true; + } + if (stream is FileStream file && file.CanSeek) + { + source = new FileByteSource(file.SafeFileHandle, file.Length, owner); + return true; + } + if (stream.CanSeek) + { + source = new StreamByteSource(stream, leaveOpen); + return true; + } + source = null; + return false; + } + } +} diff --git a/src/ExcelReader.Core/Reader/Sources/FileByteSource.cs b/src/ExcelReader.Core/Reader/Sources/FileByteSource.cs new file mode 100644 index 00000000..7a3c54dc --- /dev/null +++ b/src/ExcelReader.Core/Reader/Sources/FileByteSource.cs @@ -0,0 +1,34 @@ +using Microsoft.Win32.SafeHandles; + +namespace ExcelReader.Core.Reader.Sources +{ + internal sealed class FileByteSource : ByteSource + { + private readonly SafeFileHandle _handle; + private readonly IDisposable? _owner; + + internal FileByteSource(SafeFileHandle handle, long length, IDisposable? owner) + { + _handle = handle; + Length = length; + _owner = owner; + } + + internal override long Length { get; } + + internal override int Read(long offset, Span destination) + { + return offset >= Length ? 0 : RandomAccess.Read(_handle, destination, offset); + } + + internal override ValueTask ReadAsync(long offset, Memory destination, CancellationToken ct) + { + return offset >= Length ? new ValueTask(0) : RandomAccess.ReadAsync(_handle, destination, offset, ct); + } + + public override void Dispose() + { + _owner?.Dispose(); + } + } +} diff --git a/src/ExcelReader.Core/Reader/Sources/MemoryByteSource.cs b/src/ExcelReader.Core/Reader/Sources/MemoryByteSource.cs new file mode 100644 index 00000000..bbf81382 --- /dev/null +++ b/src/ExcelReader.Core/Reader/Sources/MemoryByteSource.cs @@ -0,0 +1,43 @@ +namespace ExcelReader.Core.Reader.Sources +{ + internal sealed class MemoryByteSource : ByteSource + { + private readonly ReadOnlyMemory _memory; + private readonly IDisposable? _owner; + + internal MemoryByteSource(ReadOnlyMemory memory, IDisposable? owner) + { + _memory = memory; + _owner = owner; + } + + internal override long Length => _memory.Length; + + internal override int Read(long offset, Span destination) + { + if (offset < 0 || offset >= _memory.Length) + { + return 0; + } + int count = (int)Math.Min(destination.Length, _memory.Length - offset); + _memory.Span.Slice((int)offset, count).CopyTo(destination); + return count; + } + + internal override ValueTask ReadAsync(long offset, Memory destination, CancellationToken ct) + { + return new ValueTask(Read(offset, destination.Span)); + } + + internal override bool TryGetMemory(out ReadOnlyMemory memory) + { + memory = _memory; + return true; + } + + public override void Dispose() + { + _owner?.Dispose(); + } + } +} diff --git a/src/ExcelReader.Core/Reader/Sources/StreamByteSource.cs b/src/ExcelReader.Core/Reader/Sources/StreamByteSource.cs new file mode 100644 index 00000000..0a31af0e --- /dev/null +++ b/src/ExcelReader.Core/Reader/Sources/StreamByteSource.cs @@ -0,0 +1,67 @@ +namespace ExcelReader.Core.Reader.Sources +{ + /// + /// A seekable stream has one cursor, so each read takes the gate for its seek and read only; + /// decompression and parsing happen outside it. + /// + internal sealed class StreamByteSource : ByteSource + { + private readonly Stream _stream; + private readonly bool _leaveOpen; + private readonly SemaphoreSlim _gate = new(1, 1); + + internal StreamByteSource(Stream stream, bool leaveOpen) + { + _stream = stream; + _leaveOpen = leaveOpen; + Length = stream.Length; + } + + internal override long Length { get; } + + internal override int Read(long offset, Span destination) + { + if (offset >= Length) + { + return 0; + } + _gate.Wait(); + try + { + _stream.Position = offset; + return _stream.Read(destination); + } + finally + { + _gate.Release(); + } + } + + internal override async ValueTask ReadAsync(long offset, Memory destination, CancellationToken ct) + { + if (offset >= Length) + { + return 0; + } + await _gate.WaitAsync(ct).ConfigureAwait(false); + try + { + _stream.Position = offset; + return await _stream.ReadAsync(destination, ct).ConfigureAwait(false); + } + finally + { + _gate.Release(); + } + } + + public override void Dispose() + { + if (!_leaveOpen) + { + _stream.Dispose(); + } + _gate.Dispose(); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Sources/ByteSourceTests.cs b/tests/ExcelReader.Tests/Reader/Sources/ByteSourceTests.cs new file mode 100644 index 00000000..90c37ad5 --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Sources/ByteSourceTests.cs @@ -0,0 +1,192 @@ +using ExcelReader.Core.Reader.Sources; + +namespace ExcelReader.Tests.Reader.Sources +{ + public sealed class ByteSourceTests : IDisposable + { + public enum Kind + { + Memory, + ExposedMemoryStream, + OpaqueMemoryStream, + Trickle, + FileStream, + FilePath, + NonSeekable, + } + + private static readonly byte[] Payload = BuildPayload(200_000); + + private readonly string _path = Path.Combine(Path.GetTempPath(), "excelreader-bytesource-" + Guid.NewGuid().ToString("N") + ".bin"); + + public ByteSourceTests() + { + File.WriteAllBytes(_path, Payload); + } + + public void Dispose() + { + File.Delete(_path); + } + + public static TheoryData Kinds => + [ + Kind.Memory, Kind.ExposedMemoryStream, Kind.OpaqueMemoryStream, Kind.Trickle, + Kind.FileStream, Kind.FilePath, Kind.NonSeekable, + ]; + + private static byte[] BuildPayload(int length) + { + byte[] bytes = new byte[length]; + uint state = 12345; + for (int i = 0; i < length; i++) + { + state = (state * 1664525) + 1013904223; + bytes[i] = (byte)(state >> 24); + } + return bytes; + } + + private ByteSource Open(Kind kind) + { + switch (kind) + { + case Kind.Memory: + return ByteSource.FromMemory(Payload); + case Kind.ExposedMemoryStream: + MemoryStream exposed = new(); + exposed.Write(Payload); + exposed.Position = 17; + return ByteSource.FromStream(exposed, leaveOpen: false); + case Kind.OpaqueMemoryStream: + return ByteSource.FromStream(new MemoryStream(Payload, writable: false), leaveOpen: false); + case Kind.Trickle: + return ByteSource.FromStream(new TrickleStream(Payload), leaveOpen: false); + case Kind.FileStream: + return ByteSource.FromStream(File.OpenRead(_path), leaveOpen: false); + case Kind.FilePath: + return ByteSource.OpenFile(_path, asynchronous: false); + default: + return ByteSource.FromStream(new NonSeekableStream(Payload), leaveOpen: false); + } + } + + [Theory] + [MemberData(nameof(Kinds))] + public void Length_Is_The_Whole_Payload(Kind kind) + { + using ByteSource source = Open(kind); + Assert.Equal(Payload.Length, source.Length); + } + + [Theory] + [MemberData(nameof(Kinds))] + public void ReadExactly_Returns_The_Bytes_At_The_Offset(Kind kind) + { + using ByteSource source = Open(kind); + byte[] read = new byte[5000]; + source.ReadExactly(150_000, read); + Assert.True(read.AsSpan().SequenceEqual(Payload.AsSpan(150_000, 5000))); + } + + [Theory] + [MemberData(nameof(Kinds))] + public async Task ReadExactlyAsync_Returns_The_Bytes_At_The_Offset(Kind kind) + { + using ByteSource source = Open(kind); + byte[] read = new byte[5000]; + await source.ReadExactlyAsync(150_000, read, TestContext.Current.CancellationToken); + Assert.True(read.AsSpan().SequenceEqual(Payload.AsSpan(150_000, 5000))); + } + + [Theory] + [MemberData(nameof(Kinds))] + public void Read_At_The_End_Returns_Zero(Kind kind) + { + using ByteSource source = Open(kind); + Assert.Equal(0, source.Read(Payload.Length, new byte[16])); + } + + [Theory] + [MemberData(nameof(Kinds))] + public void ReadExactly_Past_The_End_Throws(Kind kind) + { + using ByteSource source = Open(kind); + Assert.Throws(() => source.ReadExactly(Payload.Length - 10, new byte[11])); + } + + [Theory] + [MemberData(nameof(Kinds))] + public void TryGetMemory_Is_True_Only_When_The_Bytes_Are_In_Memory(Kind kind) + { + using ByteSource source = Open(kind); + bool expected = kind is Kind.Memory or Kind.ExposedMemoryStream or Kind.NonSeekable; + Assert.Equal(expected, source.TryGetMemory(out ReadOnlyMemory memory)); + if (expected) + { + Assert.True(memory.Span.SequenceEqual(Payload)); + } + } + + [Theory] + [MemberData(nameof(Kinds))] + public void Concurrent_Reads_Each_Get_Their_Own_Bytes(Kind kind) + { + using ByteSource source = Open(kind); + int iterations = kind == Kind.Trickle ? 20 : 200; + Parallel.For(0, 16, worker => + { + uint state = (uint)worker + 1; + byte[] buffer = new byte[4096]; + for (int i = 0; i < iterations; i++) + { + state = (state * 1664525) + 1013904223; + int offset = (int)(state % (uint)(Payload.Length - buffer.Length)); + source.ReadExactly(offset, buffer); + Assert.True(buffer.AsSpan().SequenceEqual(Payload.AsSpan(offset, buffer.Length))); + } + }); + } + + [Fact] + public void Dispose_Closes_A_Stream_It_Owns() + { + TrickleStream stream = new(Payload); + ByteSource.FromStream(stream, leaveOpen: false).Dispose(); + Assert.False(stream.CanRead); + } + + [Fact] + public void Dispose_Leaves_A_Borrowed_Stream_Open() + { + using TrickleStream stream = new(Payload); + ByteSource.FromStream(stream, leaveOpen: true).Dispose(); + Assert.True(stream.CanRead); + } + + [Fact] + public void A_Borrowed_FileStream_Stays_Usable() + { + using FileStream stream = File.OpenRead(_path); + ByteSource.FromStream(stream, leaveOpen: true).Dispose(); + Assert.Equal(Payload[0], stream.ReadByte()); + } + + [Fact] + public void Dispose_Releases_The_File_Handle() + { + ByteSource.OpenFile(_path, asynchronous: false).Dispose(); + File.Delete(_path); + Assert.False(File.Exists(_path)); + } + + [Fact] + public async Task FromStreamAsync_Buffers_A_NonSeekable_Stream() + { + using ByteSource source = await ByteSource.FromStreamAsync( + new NonSeekableStream(Payload), leaveOpen: false, TestContext.Current.CancellationToken); + Assert.True(source.TryGetMemory(out ReadOnlyMemory memory)); + Assert.True(memory.Span.SequenceEqual(Payload)); + } + } +} diff --git a/tests/ExcelReader.Tests/TestUtils.cs b/tests/ExcelReader.Tests/TestUtils.cs index 37e41eb6..e46566ef 100644 --- a/tests/ExcelReader.Tests/TestUtils.cs +++ b/tests/ExcelReader.Tests/TestUtils.cs @@ -145,6 +145,79 @@ protected override void Dispose(bool disposing) } } + internal sealed class TrickleStream : Stream + { + private const int MaxPerRead = 7; + + private readonly byte[] _bytes; + private long _position; + private bool _disposed; + + internal TrickleStream(byte[] bytes) + { + _bytes = bytes; + } + + public override bool CanRead => !_disposed; + + public override bool CanSeek => !_disposed; + + public override bool CanWrite => false; + + public override long Length => _bytes.Length; + + public override long Position + { + get => _position; + set => _position = value; + } + + public override int Read(byte[] buffer, int offset, int count) + { + return Read(buffer.AsSpan(offset, count)); + } + + public override int Read(Span buffer) + { + ObjectDisposedException.ThrowIf(_disposed, this); + int available = (int)Math.Max(0, Math.Min(_bytes.Length - _position, Math.Min(buffer.Length, MaxPerRead))); + _bytes.AsSpan((int)_position, available).CopyTo(buffer); + _position += available; + return available; + } + + public override long Seek(long offset, SeekOrigin origin) + { + _position = origin switch + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => _position + offset, + _ => _bytes.Length + offset, + }; + return _position; + } + + public override void Flush() + { + } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotSupportedException(); + } + + protected override void Dispose(bool disposing) + { + _disposed = true; + base.Dispose(disposing); + } + } + internal sealed class TrackingStream : MemoryStream { internal TrackingStream() From a3a3752c1e7c351b533db515df43e9694493658b Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Tue, 6 Oct 2026 00:57:48 -0300 Subject: [PATCH 03/28] feat(core): add ByteSourceStream, a buffered range stream over a ByteSource Co-Authored-By: Claude Opus 5.5 --- .../Reader/Sources/ByteSourceStream.cs | 149 ++++++++++++++++++ .../Reader/Sources/ByteSourceStreamTests.cs | 113 +++++++++++++ 2 files changed, 262 insertions(+) create mode 100644 src/ExcelReader.Core/Reader/Sources/ByteSourceStream.cs create mode 100644 tests/ExcelReader.Tests/Reader/Sources/ByteSourceStreamTests.cs diff --git a/src/ExcelReader.Core/Reader/Sources/ByteSourceStream.cs b/src/ExcelReader.Core/Reader/Sources/ByteSourceStream.cs new file mode 100644 index 00000000..2d798bd0 --- /dev/null +++ b/src/ExcelReader.Core/Reader/Sources/ByteSourceStream.cs @@ -0,0 +1,149 @@ +using System.Buffers; + +namespace ExcelReader.Core.Reader.Sources +{ + /// Forward-only view of one byte range of a , with its own offset. + internal sealed class ByteSourceStream : Stream + { + private const int BufferSize = 64 * 1024; + + private readonly ByteSource _source; + private readonly long _end; + private long _next; + private byte[]? _buffer; + private int _bufferPos; + private int _bufferLen; + + internal ByteSourceStream(ByteSource source, long offset, long length) + { + _source = source; + _next = offset; + _end = offset + length; + } + + public override bool CanRead => true; + + public override bool CanSeek => false; + + public override bool CanWrite => false; + + public override long Length => throw new NotSupportedException(); + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + return Read(buffer.AsSpan(offset, count)); + } + + public override int Read(Span destination) + { + if (destination.IsEmpty) + { + return 0; + } + if (_bufferPos == _bufferLen) + { + long remaining = _end - _next; + if (remaining <= 0) + { + return 0; + } + if (destination.Length >= BufferSize) + { + int direct = _source.Read(_next, destination[..(int)Math.Min(destination.Length, remaining)]); + _next += direct; + return direct; + } + _buffer ??= ArrayPool.Shared.Rent(BufferSize); + _bufferLen = _source.Read(_next, _buffer.AsSpan(0, (int)Math.Min(BufferSize, remaining))); + _bufferPos = 0; + _next += _bufferLen; + if (_bufferLen == 0) + { + return 0; + } + } + return TakeBuffered(destination); + } + + public override async ValueTask ReadAsync(Memory destination, CancellationToken cancellationToken = default) + { + if (destination.IsEmpty) + { + return 0; + } + if (_bufferPos == _bufferLen) + { + long remaining = _end - _next; + if (remaining <= 0) + { + return 0; + } + if (destination.Length >= BufferSize) + { + int direct = await _source.ReadAsync( + _next, destination[..(int)Math.Min(destination.Length, remaining)], cancellationToken).ConfigureAwait(false); + _next += direct; + return direct; + } + _buffer ??= ArrayPool.Shared.Rent(BufferSize); + _bufferLen = await _source.ReadAsync( + _next, _buffer.AsMemory(0, (int)Math.Min(BufferSize, remaining)), cancellationToken).ConfigureAwait(false); + _bufferPos = 0; + _next += _bufferLen; + if (_bufferLen == 0) + { + return 0; + } + } + return TakeBuffered(destination.Span); + } + + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + return ReadAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask(); + } + + private int TakeBuffered(Span destination) + { + int count = Math.Min(destination.Length, _bufferLen - _bufferPos); + _buffer.AsSpan(_bufferPos, count).CopyTo(destination); + _bufferPos += count; + return count; + } + + public override void Flush() + { + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotSupportedException(); + } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotSupportedException(); + } + + protected override void Dispose(bool disposing) + { + byte[]? buffer = Interlocked.Exchange(ref _buffer, null); + if (buffer is not null) + { + ArrayPool.Shared.Return(buffer); + } + base.Dispose(disposing); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Sources/ByteSourceStreamTests.cs b/tests/ExcelReader.Tests/Reader/Sources/ByteSourceStreamTests.cs new file mode 100644 index 00000000..bc8c2bdb --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Sources/ByteSourceStreamTests.cs @@ -0,0 +1,113 @@ +using ExcelReader.Core.Reader.Sources; + +namespace ExcelReader.Tests.Reader.Sources +{ + public sealed class ByteSourceStreamTests + { + private sealed class CountingSource(byte[] bytes) : ByteSource + { + internal int Reads { get; private set; } + + internal bool Disposed { get; private set; } + + internal override long Length => bytes.Length; + + internal override int Read(long offset, Span destination) + { + Reads++; + if (offset >= bytes.Length) + { + return 0; + } + int count = (int)Math.Min(destination.Length, bytes.Length - offset); + bytes.AsSpan((int)offset, count).CopyTo(destination); + return count; + } + + internal override ValueTask ReadAsync(long offset, Memory destination, CancellationToken ct) + { + return new ValueTask(Read(offset, destination.Span)); + } + + public override void Dispose() + { + Disposed = true; + } + } + + private static byte[] Payload(int length) + { + byte[] bytes = new byte[length]; + for (int i = 0; i < length; i++) + { + bytes[i] = (byte)((i * 31) + (i >> 8)); + } + return bytes; + } + + [Fact] + public void Reads_Exactly_Its_Range() + { + byte[] bytes = Payload(300_000); + using CountingSource source = new(bytes); + using ByteSourceStream stream = new(source, offset: 1000, length: 200_000); + using MemoryStream copy = new(); + stream.CopyTo(copy); + Assert.True(copy.ToArray().AsSpan().SequenceEqual(bytes.AsSpan(1000, 200_000))); + } + + [Fact] + public async Task ReadAsync_Reads_Exactly_Its_Range() + { + byte[] bytes = Payload(300_000); + using CountingSource source = new(bytes); + await using ByteSourceStream stream = new(source, offset: 1000, length: 200_000); + using MemoryStream copy = new(); + await stream.CopyToAsync(copy, TestContext.Current.CancellationToken); + Assert.True(copy.ToArray().AsSpan().SequenceEqual(bytes.AsSpan(1000, 200_000))); + } + + [Fact] + public void Small_Reads_Are_Served_From_One_Buffered_Fetch() + { + using CountingSource source = new(Payload(60_000)); + using ByteSourceStream stream = new(source, offset: 0, length: 60_000); + byte[] chunk = new byte[100]; + for (int i = 0; i < 600; i++) + { + stream.ReadExactly(chunk); + } + Assert.Equal(1, source.Reads); + } + + [Fact] + public void A_Large_Read_Bypasses_The_Buffer() + { + byte[] bytes = Payload(300_000); + using CountingSource source = new(bytes); + using ByteSourceStream stream = new(source, offset: 0, length: 300_000); + byte[] big = new byte[300_000]; + Assert.Equal(300_000, stream.Read(big)); + Assert.Equal(1, source.Reads); + Assert.True(big.AsSpan().SequenceEqual(bytes)); + } + + [Fact] + public void A_Source_Shorter_Than_The_Range_Ends_The_Stream() + { + using CountingSource source = new(Payload(100)); + using ByteSourceStream stream = new(source, offset: 50, length: 500); + byte[] buffer = new byte[1000]; + Assert.Equal(50, stream.Read(buffer)); + Assert.Equal(0, stream.Read(buffer)); + } + + [Fact] + public void Dispose_Does_Not_Dispose_The_Source() + { + using CountingSource source = new(Payload(10)); + new ByteSourceStream(source, 0, 10).Dispose(); + Assert.False(source.Disposed); + } + } +} From 702e21a8ed8ddbee83e88a5b4c72adf254c679a8 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Tue, 6 Oct 2026 01:05:52 -0300 Subject: [PATCH 04/28] refactor(core): rename ZipMemoryIndex to ZipIndex and index names into the central directory Co-Authored-By: Claude Opus 5.5 --- src/ExcelReader.Core/Reader/Excel.cs | 12 ++-- .../Reader/Internal/WorkbookLookups.cs | 2 +- .../Reader/Xlsb/XlsbReader.Memory.cs | 8 +-- .../Reader/Xlsb/XlsbReader.cs | 4 +- .../Reader/Xlsx/XlsxReader.Memory.cs | 6 +- .../Reader/Xlsx/XlsxReader.cs | 4 +- .../Zip/{ZipMemoryIndex.cs => ZipIndex.cs} | 47 +++++++------- .../Reader/Zip/ZipReaderOpen.cs | 2 +- .../ZipIndexTests.cs} | 62 +++++++++---------- 9 files changed, 74 insertions(+), 73 deletions(-) rename src/ExcelReader.Core/Reader/Zip/{ZipMemoryIndex.cs => ZipIndex.cs} (92%) rename tests/ExcelReader.Tests/Reader/{Xlsx/ZipMemoryIndexTests.cs => Zip/ZipIndexTests.cs} (91%) diff --git a/src/ExcelReader.Core/Reader/Excel.cs b/src/ExcelReader.Core/Reader/Excel.cs index fbbe7e4d..bbca4065 100644 --- a/src/ExcelReader.Core/Reader/Excel.cs +++ b/src/ExcelReader.Core/Reader/Excel.cs @@ -351,7 +351,7 @@ public static IExcelRowReader Open(Stream stream, bool leaveOpen = true, ExcelRe /// /// Opens a workbook from an in-memory buffer, auto-detecting its format (XLSX/XLSB/XLS) from its - /// signature. XLSX/XLSB route through instead of + /// signature. XLSX/XLSB route through instead of /// a /, so the returned reader never /// suspends, even under await foreach. /// @@ -367,7 +367,7 @@ public static IExcelRowReader Open(ReadOnlyMemory data, ExcelReaderOptions ReadOnlyMemory plain = EncryptedPackageOpener.DecryptToMemory(data, effective); return OpenFromPlainMemory(plain, effective); } - ExcelFileFormat format = ClassifyMemory(data, effective, out ZipMemoryIndex? memZip); + ExcelFileFormat format = ClassifyMemory(data, effective, out ZipIndex? memZip); if (format is ExcelFileFormat.Unknown) { memZip?.Dispose(); @@ -384,7 +384,7 @@ public static IExcelRowReader Open(ReadOnlyMemory data, ExcelReaderOptions private static IExcelRowReader OpenFromPlainMemory(ReadOnlyMemory plain, ExcelReaderOptions options) { - ExcelFileFormat format = ClassifyMemory(plain, options, out ZipMemoryIndex? memZip); + ExcelFileFormat format = ClassifyMemory(plain, options, out ZipIndex? memZip); if (format is not (ExcelFileFormat.Xlsb or ExcelFileFormat.Xlsx)) { memZip?.Dispose(); @@ -398,7 +398,7 @@ private static IExcelRowReader OpenFromPlainMemory(ReadOnlyMemory plain, E }; } - private static ExcelFileFormat ClassifyMemory(ReadOnlyMemory data, ExcelReaderOptions options, out ZipMemoryIndex? memZip) + private static ExcelFileFormat ClassifyMemory(ReadOnlyMemory data, ExcelReaderOptions options, out ZipIndex? memZip) { memZip = null; ReadOnlySpan span = data.Span; @@ -413,7 +413,7 @@ private static ExcelFileFormat ClassifyMemory(ReadOnlyMemory data, ExcelRe ? ExcelFileFormat.EncryptedOoxml : ExcelFileFormat.Xls; } - memZip = ZipMemoryIndex.Create(data, options); + memZip = ZipIndex.Create(data, options); return memZip.TryGetEntry("xl/workbook.bin"u8, out _) ? ExcelFileFormat.Xlsb : ExcelFileFormat.Xlsx; } @@ -640,7 +640,7 @@ public static ExcelFileFormat DetectFileFormat(Stream stream) /// The detected , or if the signature matches no supported format. public static ExcelFileFormat DetectFileFormat(ReadOnlyMemory data) { - var format = ClassifyMemory(data, ExcelReaderOptions.Default, out ZipMemoryIndex? memZip); + var format = ClassifyMemory(data, ExcelReaderOptions.Default, out ZipIndex? memZip); memZip?.Dispose(); return format; } diff --git a/src/ExcelReader.Core/Reader/Internal/WorkbookLookups.cs b/src/ExcelReader.Core/Reader/Internal/WorkbookLookups.cs index e414e0c5..d4290602 100644 --- a/src/ExcelReader.Core/Reader/Internal/WorkbookLookups.cs +++ b/src/ExcelReader.Core/Reader/Internal/WorkbookLookups.cs @@ -73,7 +73,7 @@ internal static ZipArchiveEntry GetWorksheetEntry(ZipArchive zip, string path) } [SkipLocalsInit] - internal static ZipEntryRef GetWorksheetEntry(ZipMemoryIndex memZip, string path) + internal static ZipEntryRef GetWorksheetEntry(ZipIndex memZip, string path) { Span stackBuffer = stackalloc byte[256]; ReadOnlySpan utf8Path = Encoding.UTF8.GetByteCount(path) <= stackBuffer.Length diff --git a/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Memory.cs b/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Memory.cs index a2b18a18..14da0cee 100644 --- a/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Memory.cs +++ b/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Memory.cs @@ -8,15 +8,15 @@ public sealed partial class XlsbReader internal static XlsbReader CreateFromMemory(ReadOnlyMemory data, ExcelReaderOptions? options = null) { ExcelReaderOptions effectiveOptions = options ?? ExcelReaderOptions.Default; - return CreateFromMemory(ZipMemoryIndex.Create(data, effectiveOptions), effectiveOptions); + return CreateFromMemory(ZipIndex.Create(data, effectiveOptions), effectiveOptions); } - internal static XlsbReader CreateFromMemory(ZipMemoryIndex memZip, ExcelReaderOptions effectiveOptions) + internal static XlsbReader CreateFromMemory(ZipIndex memZip, ExcelReaderOptions effectiveOptions) { return ZipReaderOpen.FromMemory(memZip, zip => BuildFromMemory(zip, effectiveOptions)); } - private static XlsbReader BuildFromMemory(ZipMemoryIndex memZip, ExcelReaderOptions effectiveOptions) + private static XlsbReader BuildFromMemory(ZipIndex memZip, ExcelReaderOptions effectiveOptions) { DecompressedByteCounter decompressedBytes = new(effectiveOptions.MaxTotalDecompressedBytes); using ZipPart wbPart = memZip.OpenPartOrDefault("xl/workbook.bin"u8, decompressedBytes); @@ -34,7 +34,7 @@ private static XlsbReader BuildFromMemory(ZipMemoryIndex memZip, ExcelReaderOpti } private static (byte[] Flat, int[] Offsets) LoadSharedStringsFromMemory( - ZipMemoryIndex memZip, DecompressedByteCounter decompressedBytes, ExcelReaderOptions options) + ZipIndex memZip, DecompressedByteCounter decompressedBytes, ExcelReaderOptions options) { if (!memZip.TryGetEntry("xl/sharedStrings.bin"u8, out ZipEntryRef entry)) { diff --git a/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.cs b/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.cs index 0eb69b3c..0c81dfdc 100644 --- a/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.cs +++ b/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.cs @@ -22,7 +22,7 @@ public sealed partial class XlsbReader : IExcelRowReader, IExcelRowReader data, ExcelReaderOptions? options = null) { ExcelReaderOptions effectiveOptions = options ?? ExcelReaderOptions.Default; - return CreateFromMemory(ZipMemoryIndex.Create(data, effectiveOptions), effectiveOptions); + return CreateFromMemory(ZipIndex.Create(data, effectiveOptions), effectiveOptions); } - internal static XlsxReader CreateFromMemory(ZipMemoryIndex memZip, ExcelReaderOptions effectiveOptions) + internal static XlsxReader CreateFromMemory(ZipIndex memZip, ExcelReaderOptions effectiveOptions) { return ZipReaderOpen.FromMemory(memZip, zip => BuildFromMemory(zip, effectiveOptions)); } - private static XlsxReader BuildFromMemory(ZipMemoryIndex memZip, ExcelReaderOptions effectiveOptions) + private static XlsxReader BuildFromMemory(ZipIndex memZip, ExcelReaderOptions effectiveOptions) { DecompressedByteCounter decompressedBytes = new(effectiveOptions.MaxTotalDecompressedBytes); using ZipPart wbPart = memZip.OpenPartOrDefault("xl/workbook.xml"u8, decompressedBytes); diff --git a/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.cs b/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.cs index 5c9c9792..220c6cee 100644 --- a/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.cs +++ b/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.cs @@ -11,7 +11,7 @@ public sealed partial class XlsxReader : IExcelRowReader, IExcelRowReader EocdSignatureBytes => [0x50, 0x4B, 0x05, 0x06]; private readonly ReadOnlyMemory _file; + private readonly ReadOnlyMemory _directory; private readonly ZipEntryRef[] _entries; private bool _disposed; - private ZipMemoryIndex(ReadOnlyMemory file, ZipEntryRef[] entries, int count) + private ZipIndex(ReadOnlyMemory file, ReadOnlyMemory directory, ZipEntryRef[] entries, int count) { _file = file; + _directory = directory; _entries = entries; Count = count; } internal int Count { get; } - internal static ZipMemoryIndex Create(ReadOnlyMemory file, ExcelReaderOptions options) + internal static ZipIndex Create(ReadOnlyMemory file, ExcelReaderOptions options) { ReadOnlySpan span = file.Span; long eocdOffset = FindEocd(span); @@ -83,24 +85,25 @@ internal static ZipMemoryIndex Create(ReadOnlyMemory file, ExcelReaderOpti { throw new InvalidDataException("The ZIP central directory is out of range."); } + ReadOnlyMemory directory = file.Slice((int)cdOffset, (int)cdSize); long maxHint = Math.Max(16, options.MaxZipEntries > 0 ? options.MaxZipEntries : 65_536); ZipEntryRef[] entries = ArrayPool.Shared.Rent((int)Math.Clamp(declaredCount, 16, maxHint)); int count; try { - count = WalkCentralDirectory(span, cdOffset, cdSize, ref entries, options); - ThrowIfDuplicateEntryNames(file, entries, count); + count = WalkCentralDirectory(directory.Span, ref entries, options); + ThrowIfDuplicateEntryNames(directory, entries, count); } catch { ArrayPool.Shared.Return(entries); throw; } - return new ZipMemoryIndex(file, entries, count); + return new ZipIndex(file, directory, entries, count); } - private static void ThrowIfDuplicateEntryNames(ReadOnlyMemory file, ZipEntryRef[] entries, int count) + private static void ThrowIfDuplicateEntryNames(ReadOnlyMemory directory, ZipEntryRef[] entries, int count) { if (count <= 1) { @@ -115,18 +118,18 @@ private static void ThrowIfDuplicateEntryNames(ReadOnlyMemory file, ZipEnt } Array.Sort(order, 0, count, Comparer.Create((a, b) => { - ReadOnlySpan span = file.Span; + ReadOnlySpan span = directory.Span; ref readonly ZipEntryRef ea = ref entries[a]; ref readonly ZipEntryRef eb = ref entries[b]; return span.Slice(ea.NameStart, ea.NameLength).SequenceCompareTo(span.Slice(eb.NameStart, eb.NameLength)); })); - ReadOnlySpan fileSpan = file.Span; + ReadOnlySpan directorySpan = directory.Span; for (int i = 1; i < count; i++) { ref readonly ZipEntryRef prev = ref entries[order[i - 1]]; ref readonly ZipEntryRef curr = ref entries[order[i]]; if (prev.NameLength == curr.NameLength && - fileSpan.Slice(prev.NameStart, prev.NameLength).SequenceEqual(fileSpan.Slice(curr.NameStart, curr.NameLength))) + directorySpan.Slice(prev.NameStart, prev.NameLength).SequenceEqual(directorySpan.Slice(curr.NameStart, curr.NameLength))) { throw new InvalidDataException("The ZIP central directory contains a duplicate entry name."); } @@ -140,10 +143,10 @@ private static void ThrowIfDuplicateEntryNames(ReadOnlyMemory file, ZipEnt internal bool TryGetEntry(ReadOnlySpan utf8Name, out ZipEntryRef entry) { - ReadOnlySpan fileSpan = _file.Span; + ReadOnlySpan directory = _directory.Span; foreach (ref readonly ZipEntryRef candidate in _entries.AsSpan(0, Count)) { - if (fileSpan.Slice(candidate.NameStart, candidate.NameLength).SequenceEqual(utf8Name)) + if (directory.Slice(candidate.NameStart, candidate.NameLength).SequenceEqual(utf8Name)) { entry = candidate; return true; @@ -298,11 +301,10 @@ private static (long CdOffset, long CdSize, long Count) ReadZip64Eocd(ReadOnlySp return (cdOffset, cdSize, count); } - private static int WalkCentralDirectory(ReadOnlySpan span, long cdOffset, long cdSize, ref ZipEntryRef[] entries, ExcelReaderOptions options) + private static int WalkCentralDirectory(ReadOnlySpan directory, ref ZipEntryRef[] entries, ExcelReaderOptions options) { - // Create has checked cdOffset <= span.Length - cdSize, so end <= span.Length and cannot overflow. - long end = cdOffset + cdSize; - long pos = cdOffset; + int end = directory.Length; + int pos = 0; int count = 0; while (pos < end) { @@ -310,7 +312,7 @@ private static int WalkCentralDirectory(ReadOnlySpan span, long cdOffset, { throw new InvalidDataException("The ZIP central directory is truncated."); } - ReadOnlySpan record = span.Slice((int)pos, CentralDirectoryFixedSize); + ReadOnlySpan record = directory.Slice(pos, CentralDirectoryFixedSize); if (BinaryPrimitives.ReadInt32LittleEndian(record) != CentralDirectorySignature) { throw new InvalidDataException("Invalid ZIP central directory record signature."); @@ -321,21 +323,20 @@ private static int WalkCentralDirectory(ReadOnlySpan span, long cdOffset, throw new NotSupportedException("Encrypted ZIP entries are not supported."); } - long nameStart = pos + CentralDirectoryFixedSize; + int nameStart = pos + CentralDirectoryFixedSize; int variableLength = fields.NameLength + fields.ExtraLength + fields.CommentLength; if (variableLength > end - nameStart) { throw new InvalidDataException("The ZIP central directory is truncated."); } - long recordEnd = nameStart + variableLength; (long compressed, long uncompressed, long localOffset) = ResolveZip64Sizes( - span, (int)(nameStart + fields.NameLength), fields.ExtraLength, fields); + directory, nameStart + fields.NameLength, fields.ExtraLength, fields); EnsureCapacity(ref entries, count); entries[count] = new ZipEntryRef { - NameStart = (int)nameStart, + NameStart = nameStart, NameLength = fields.NameLength, LocalHeaderOffset = localOffset, CompressedSize = compressed, @@ -345,7 +346,7 @@ private static int WalkCentralDirectory(ReadOnlySpan span, long cdOffset, }; count++; LimitChecks.ThrowIfTooManyEntries(count, options); - pos = recordEnd; + pos = nameStart + variableLength; } return count; } @@ -469,7 +470,7 @@ private long ResolveDataOffset(in ZipEntryRef entry) throw new InvalidDataException("The ZIP local file header name runs past the end of the file."); } if (nameLength != entry.NameLength || - !fileSpan.Slice((int)nameStart, nameLength).SequenceEqual(fileSpan.Slice(entry.NameStart, entry.NameLength))) + !fileSpan.Slice((int)nameStart, nameLength).SequenceEqual(_directory.Span.Slice(entry.NameStart, entry.NameLength))) { throw new InvalidDataException("The ZIP local file header name does not match the central directory."); } diff --git a/src/ExcelReader.Core/Reader/Zip/ZipReaderOpen.cs b/src/ExcelReader.Core/Reader/Zip/ZipReaderOpen.cs index a9430d9e..3a08afc1 100644 --- a/src/ExcelReader.Core/Reader/Zip/ZipReaderOpen.cs +++ b/src/ExcelReader.Core/Reader/Zip/ZipReaderOpen.cs @@ -65,7 +65,7 @@ internal static async ValueTask FromOpenZipAsync( } } - internal static TResult FromMemory(ZipMemoryIndex memZip, Func build) + internal static TResult FromMemory(ZipIndex memZip, Func build) { try { diff --git a/tests/ExcelReader.Tests/Reader/Xlsx/ZipMemoryIndexTests.cs b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs similarity index 91% rename from tests/ExcelReader.Tests/Reader/Xlsx/ZipMemoryIndexTests.cs rename to tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs index 584c76df..e1e48226 100644 --- a/tests/ExcelReader.Tests/Reader/Xlsx/ZipMemoryIndexTests.cs +++ b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs @@ -6,9 +6,9 @@ using ExcelReader.Core.Reader; using ExcelReader.Core.Reader.Zip; -namespace ExcelReader.Tests.Reader.Xlsx +namespace ExcelReader.Tests.Reader.Zip { - public class ZipMemoryIndexTests + public class ZipIndexTests { [Fact] public void OpenPartOnDeflatedEntryMatchesStreamedRead() @@ -17,7 +17,7 @@ public void OpenPartOnDeflatedEntryMatchesStreamedRead() byte[] zipBytes = built.ToArray(); byte[] expected = ReadViaZipArchive(zipBytes, "xl/workbook.xml"); - using ZipMemoryIndex index = ZipMemoryIndex.Create(zipBytes, ExcelReaderOptions.Default); + using ZipIndex index = ZipIndex.Create(zipBytes, ExcelReaderOptions.Default); Assert.True(index.TryGetEntry("xl/workbook.xml"u8, out ZipEntryRef entry)); Assert.Equal((ushort)8, entry.Method); @@ -31,7 +31,7 @@ public void OpenPartOnStoredEntryAliasesTheSourceBufferWithoutCopying() byte[] payload = Encoding.UTF8.GetBytes("hello world, stored and not deflated"); byte[] zipBytes = BuildZipWithOneEntry("hello.txt", payload, CompressionLevel.NoCompression); - using ZipMemoryIndex index = ZipMemoryIndex.Create(zipBytes, ExcelReaderOptions.Default); + using ZipIndex index = ZipIndex.Create(zipBytes, ExcelReaderOptions.Default); Assert.True(index.TryGetEntry("hello.txt"u8, out ZipEntryRef entry)); Assert.Equal((ushort)0, entry.Method); @@ -47,7 +47,7 @@ public void TryGetEntryReturnsFalseForAnAbsentName() using MemoryStream built = WorkbookBuilder.Build("""1"""); byte[] zipBytes = built.ToArray(); - using ZipMemoryIndex index = ZipMemoryIndex.Create(zipBytes, ExcelReaderOptions.Default); + using ZipIndex index = ZipIndex.Create(zipBytes, ExcelReaderOptions.Default); Assert.False(index.TryGetEntry("xl/doesNotExist.xml"u8, out _)); } @@ -69,7 +69,7 @@ public void CreateThrowsOnDuplicateCentralDirectoryEntryNames() byte[] zipBytes = ms.ToArray(); InvalidDataException ex = Assert.Throws( - () => ZipMemoryIndex.Create(zipBytes, ExcelReaderOptions.Default)); + () => ZipIndex.Create(zipBytes, ExcelReaderOptions.Default)); Assert.Contains("duplicate", ex.Message, StringComparison.OrdinalIgnoreCase); } @@ -80,7 +80,7 @@ public void OpenPartThrowsWhenLocalHeaderNameDoesNotMatchCentralDirectory() byte[] zipBytes = BuildZipWithOneEntry("hello.txt", payload, CompressionLevel.NoCompression); long localHeaderOffset; - using (ZipMemoryIndex index = ZipMemoryIndex.Create(zipBytes, ExcelReaderOptions.Default)) + using (ZipIndex index = ZipIndex.Create(zipBytes, ExcelReaderOptions.Default)) { Assert.True(index.TryGetEntry("hello.txt"u8, out ZipEntryRef entry)); localHeaderOffset = entry.LocalHeaderOffset; @@ -90,7 +90,7 @@ public void OpenPartThrowsWhenLocalHeaderNameDoesNotMatchCentralDirectory() Assert.Equal((byte)'h', zipBytes[nameOffset]); zipBytes[nameOffset] = (byte)'j'; - using ZipMemoryIndex mutatedIndex = ZipMemoryIndex.Create(zipBytes, ExcelReaderOptions.Default); + using ZipIndex mutatedIndex = ZipIndex.Create(zipBytes, ExcelReaderOptions.Default); Assert.True(mutatedIndex.TryGetEntry("hello.txt"u8, out ZipEntryRef mutatedEntry)); InvalidDataException ex = Assert.Throws( () => mutatedIndex.OpenPart(mutatedEntry, new DecompressedByteCounter(0))); @@ -104,7 +104,7 @@ public void OpenPartThrowsWhenDeclaredSizeExceedsTheRemainingBudget() byte[] zipBytes = built.ToArray(); PatchCentralDirectoryUInt32(zipBytes, "xl/workbook.xml", fieldOffset: 24, 50_000_000); - using ZipMemoryIndex index = ZipMemoryIndex.Create(zipBytes, ExcelReaderOptions.Default); + using ZipIndex index = ZipIndex.Create(zipBytes, ExcelReaderOptions.Default); Assert.True(index.TryGetEntry("xl/workbook.xml"u8, out ZipEntryRef entry)); var counter = new DecompressedByteCounter(4096, nameof(ExcelReaderOptions.MaxTotalDecompressedBytes)); @@ -131,7 +131,7 @@ public void CreateThrowsWhenEntryCountExceedsMaxZipEntries() var options = new ExcelReaderOptions { MaxZipEntries = 10 }; ExcelLimitExceededException ex = Assert.Throws( - () => ZipMemoryIndex.Create(zipBytes, options)); + () => ZipIndex.Create(zipBytes, options)); Assert.Equal(nameof(ExcelReaderOptions.MaxZipEntries), ex.LimitName); } @@ -142,7 +142,7 @@ public void CreateThrowsForAnEncryptedEntry() byte[] zipBytes = built.ToArray(); PatchCentralDirectoryUInt16(zipBytes, "xl/workbook.xml", fieldOffset: 8, 0x0001); - Assert.Throws(() => ZipMemoryIndex.Create(zipBytes, ExcelReaderOptions.Default)); + Assert.Throws(() => ZipIndex.Create(zipBytes, ExcelReaderOptions.Default)); } [Fact] @@ -152,7 +152,7 @@ public void OpenPartThrowsForAnUnsupportedCompressionMethod() byte[] zipBytes = built.ToArray(); PatchCentralDirectoryUInt16(zipBytes, "xl/workbook.xml", fieldOffset: 10, 12); - using ZipMemoryIndex index = ZipMemoryIndex.Create(zipBytes, ExcelReaderOptions.Default); + using ZipIndex index = ZipIndex.Create(zipBytes, ExcelReaderOptions.Default); Assert.True(index.TryGetEntry("xl/workbook.xml"u8, out ZipEntryRef entry)); Assert.Throws(() => index.OpenPart(entry, new DecompressedByteCounter(0))); } @@ -161,7 +161,7 @@ public void OpenPartThrowsForAnUnsupportedCompressionMethod() public void CreateThrowsWhenNoEndOfCentralDirectoryRecordExists() { byte[] notAZip = Encoding.UTF8.GetBytes("this is not a zip file at all"); - Assert.Throws(() => ZipMemoryIndex.Create(notAZip, ExcelReaderOptions.Default)); + Assert.Throws(() => ZipIndex.Create(notAZip, ExcelReaderOptions.Default)); } [Fact] @@ -205,7 +205,7 @@ public void MutatedZipBytesNeverCrashTheMemoryIndex() private static void OpenAllPartsAndDrain(byte[] bytes) { - using ZipMemoryIndex index = ZipMemoryIndex.Create(bytes, ExcelReaderOptions.Default); + using ZipIndex index = ZipIndex.Create(bytes, ExcelReaderOptions.Default); var counter = new DecompressedByteCounter(ExcelReaderOptions.Default.MaxTotalDecompressedBytes); foreach (string name in (string[])["xl/workbook.xml", "xl/sharedStrings.xml", "xl/styles.xml", "xl/worksheets/sheet1.xml"]) { @@ -243,7 +243,7 @@ public void HugeZip64EocdLocatorOffsetThrowsInsteadOfWrapping() WriteEocd(bytes, 20, declaredCount: 0xFFFF, cdSize: Zip64SentinelU32, cdOffset: Zip64SentinelU32); InvalidDataException ex = Assert.Throws( - () => ZipMemoryIndex.Create(bytes, ExcelReaderOptions.Default)); + () => ZipIndex.Create(bytes, ExcelReaderOptions.Default)); Assert.Contains("ZIP64", ex.Message, StringComparison.Ordinal); } @@ -259,7 +259,7 @@ public void HugeZip64CentralDirectorySizeThrowsInsteadOfWrapping() WriteEocd(bytes, 76, declaredCount: 0xFFFF, cdSize: Zip64SentinelU32, cdOffset: Zip64SentinelU32); InvalidDataException ex = Assert.Throws( - () => ZipMemoryIndex.Create(bytes, ExcelReaderOptions.Default)); + () => ZipIndex.Create(bytes, ExcelReaderOptions.Default)); Assert.Contains("central directory", ex.Message, StringComparison.OrdinalIgnoreCase); } @@ -297,7 +297,7 @@ public void NegativeZip64UncompressedSizeThrowsInvalidDataException() byte[] bytes = BuildCdWithZip64Field(compressedSize: 5, uncompressedSize: Zip64SentinelU32, zip64Value: -1); InvalidDataException ex = Assert.Throws( - () => ZipMemoryIndex.Create(bytes, ExcelReaderOptions.Default)); + () => ZipIndex.Create(bytes, ExcelReaderOptions.Default)); Assert.Contains("negative", ex.Message, StringComparison.OrdinalIgnoreCase); } @@ -307,7 +307,7 @@ public void NegativeZip64CompressedSizeThrowsInvalidDataException() byte[] bytes = BuildCdWithZip64Field(compressedSize: Zip64SentinelU32, uncompressedSize: 5, zip64Value: -1); InvalidDataException ex = Assert.Throws( - () => ZipMemoryIndex.Create(bytes, ExcelReaderOptions.Default)); + () => ZipIndex.Create(bytes, ExcelReaderOptions.Default)); Assert.Contains("negative", ex.Message, StringComparison.OrdinalIgnoreCase); } @@ -318,7 +318,7 @@ public void ValidZip64ArchiveOpensTheStoredEntry() { Zip64Layout zip = BuildStoredZip64(Zip64Payload); - using ZipMemoryIndex index = ZipMemoryIndex.Create(zip.Bytes, ExcelReaderOptions.Default); + using ZipIndex index = ZipIndex.Create(zip.Bytes, ExcelReaderOptions.Default); Assert.True(index.TryGetEntry("a.txt"u8, out ZipEntryRef entry)); Assert.Equal(0, entry.LocalHeaderOffset); Assert.Equal(Zip64Payload.Length, entry.CompressedSize); @@ -344,7 +344,7 @@ public void HugeZip64LocalHeaderOffsetThrowsInvalidDataOnOpen(long localOffset) { Zip64Layout zip = BuildStoredZip64(Zip64Payload, localOffset: localOffset); - using ZipMemoryIndex index = ZipMemoryIndex.Create(zip.Bytes, ExcelReaderOptions.Default); + using ZipIndex index = ZipIndex.Create(zip.Bytes, ExcelReaderOptions.Default); Assert.True(index.TryGetEntry("a.txt"u8, out ZipEntryRef entry)); Assert.Equal(localOffset, entry.LocalHeaderOffset); InvalidDataException ex = Assert.Throws( @@ -358,7 +358,7 @@ public void LocalHeaderOffsetJustInsideTheFileEndThrowsInvalidDataOnOpen() Zip64Layout probe = BuildStoredZip64(Zip64Payload); Zip64Layout zip = BuildStoredZip64(Zip64Payload, localOffset: probe.Bytes.Length - 29); - using ZipMemoryIndex index = ZipMemoryIndex.Create(zip.Bytes, ExcelReaderOptions.Default); + using ZipIndex index = ZipIndex.Create(zip.Bytes, ExcelReaderOptions.Default); Assert.True(index.TryGetEntry("a.txt"u8, out ZipEntryRef entry)); InvalidDataException ex = Assert.Throws( () => index.OpenPart(entry, new DecompressedByteCounter(0))); @@ -374,7 +374,7 @@ public void HugeZip64CompressedSizeThrowsInvalidDataOnOpen(long compressedSize) { Zip64Layout zip = BuildStoredZip64(Zip64Payload, compressedSize: compressedSize); - using ZipMemoryIndex index = ZipMemoryIndex.Create(zip.Bytes, ExcelReaderOptions.Default); + using ZipIndex index = ZipIndex.Create(zip.Bytes, ExcelReaderOptions.Default); Assert.True(index.TryGetEntry("a.txt"u8, out ZipEntryRef entry)); Assert.Equal(compressedSize, entry.CompressedSize); InvalidDataException ex = Assert.Throws( @@ -391,7 +391,7 @@ public void CompressedSizeEndingExactlyAtTheFileEndIsAcceptedAndOneMoreIsRejecte long exact = probe.Bytes.Length - probe.DataOffset; Zip64Layout fits = BuildStoredZip64(Zip64Payload, compressedSize: exact); - using (ZipMemoryIndex index = ZipMemoryIndex.Create(fits.Bytes, ExcelReaderOptions.Default)) + using (ZipIndex index = ZipIndex.Create(fits.Bytes, ExcelReaderOptions.Default)) { Assert.True(index.TryGetEntry("a.txt"u8, out ZipEntryRef entry)); using ZipPart part = index.OpenPart(entry, new DecompressedByteCounter(0)); @@ -399,7 +399,7 @@ public void CompressedSizeEndingExactlyAtTheFileEndIsAcceptedAndOneMoreIsRejecte } Zip64Layout overruns = BuildStoredZip64(Zip64Payload, compressedSize: exact + 1); - using (ZipMemoryIndex index = ZipMemoryIndex.Create(overruns.Bytes, ExcelReaderOptions.Default)) + using (ZipIndex index = ZipIndex.Create(overruns.Bytes, ExcelReaderOptions.Default)) { Assert.True(index.TryGetEntry("a.txt"u8, out ZipEntryRef entry)); Assert.Throws(() => index.OpenPart(entry, new DecompressedByteCounter(0))); @@ -414,7 +414,7 @@ public void Zip64LocalHeaderOffsetAboveLongMaxIsRejectedAsNegative(ulong localOf Zip64Layout zip = BuildStoredZip64(Zip64Payload, localOffset: unchecked((long)localOffset)); InvalidDataException ex = Assert.Throws( - () => ZipMemoryIndex.Create(zip.Bytes, ExcelReaderOptions.Default)); + () => ZipIndex.Create(zip.Bytes, ExcelReaderOptions.Default)); Assert.Contains("negative", ex.Message, StringComparison.OrdinalIgnoreCase); } @@ -427,7 +427,7 @@ public void Zip64LocatorOffsetAboveLongMaxIsRejectedNotIgnored(ulong zip64EocdOf BinaryPrimitives.WriteUInt64LittleEndian(zip.Bytes.AsSpan(zip.LocatorOffset + 8), zip64EocdOffset); InvalidDataException ex = Assert.Throws( - () => ZipMemoryIndex.Create(zip.Bytes, ExcelReaderOptions.Default)); + () => ZipIndex.Create(zip.Bytes, ExcelReaderOptions.Default)); Assert.Contains("ZIP64", ex.Message, StringComparison.Ordinal); } @@ -446,7 +446,7 @@ public void Zip64CentralDirectoryBoundsThatWouldOverflowAreRejected(long cdOffse BinaryPrimitives.WriteInt64LittleEndian(zip.Bytes.AsSpan(zip.Zip64EocdOffset + 48), cdOffset); InvalidDataException ex = Assert.Throws( - () => ZipMemoryIndex.Create(zip.Bytes, ExcelReaderOptions.Default)); + () => ZipIndex.Create(zip.Bytes, ExcelReaderOptions.Default)); Assert.Contains("central directory", ex.Message, StringComparison.OrdinalIgnoreCase); } @@ -459,7 +459,7 @@ public void Zip64CentralDirectorySizeOneByteLongerThanItsRecordsIsRejected() Assert.Equal(zip.Zip64EocdOffset, cdOffset + cdSize); BinaryPrimitives.WriteInt64LittleEndian(zip.Bytes.AsSpan(zip.Zip64EocdOffset + 40), cdSize + 1); - Assert.Throws(() => ZipMemoryIndex.Create(zip.Bytes, ExcelReaderOptions.Default)); + Assert.Throws(() => ZipIndex.Create(zip.Bytes, ExcelReaderOptions.Default)); } [Theory] @@ -473,7 +473,7 @@ public void CentralDirectoryVariableFieldsRunningPastTheDirectoryEndAreRejected( PatchCentralDirectoryUInt16(zipBytes, "hello.txt", fieldOffset, value); InvalidDataException ex = Assert.Throws( - () => ZipMemoryIndex.Create(zipBytes, ExcelReaderOptions.Default)); + () => ZipIndex.Create(zipBytes, ExcelReaderOptions.Default)); Assert.Contains("truncated", ex.Message, StringComparison.OrdinalIgnoreCase); } @@ -488,7 +488,7 @@ public void CentralDirectorySizeShorterThanItsFixedRecordIsRejected(uint cdSize) WriteEocd(zipBytes, eocd, declaredCount: 1, cdSize: cdSize, cdOffset: cdOffset); InvalidDataException ex = Assert.Throws( - () => ZipMemoryIndex.Create(zipBytes, ExcelReaderOptions.Default)); + () => ZipIndex.Create(zipBytes, ExcelReaderOptions.Default)); Assert.Contains("truncated", ex.Message, StringComparison.OrdinalIgnoreCase); } @@ -498,7 +498,7 @@ public void LocalExtraLengthPushingDataPastTheFileEndThrowsInvalidDataOnOpen() byte[] zipBytes = BuildZipWithOneEntry("hello.txt", Zip64Payload, CompressionLevel.NoCompression); BinaryPrimitives.WriteUInt16LittleEndian(zipBytes.AsSpan(28), 0xFFFF); - using ZipMemoryIndex index = ZipMemoryIndex.Create(zipBytes, ExcelReaderOptions.Default); + using ZipIndex index = ZipIndex.Create(zipBytes, ExcelReaderOptions.Default); Assert.True(index.TryGetEntry("hello.txt"u8, out ZipEntryRef entry)); InvalidDataException ex = Assert.Throws( () => index.OpenPart(entry, new DecompressedByteCounter(0))); From 40b7a0403365be036bdf08a093bce6203490ea80 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Tue, 6 Oct 2026 01:11:09 -0300 Subject: [PATCH 05/28] feat(core): let ZipIndex read its directory and entries through any ByteSource Co-Authored-By: Claude Opus 5.5 --- src/ExcelReader.Core/Reader/Zip/ZipIndex.cs | 457 +++++++++++++++--- .../Reader/Zip/ZipIndexSourceTests.cs | 306 ++++++++++++ 2 files changed, 694 insertions(+), 69 deletions(-) create mode 100644 tests/ExcelReader.Tests/Reader/Zip/ZipIndexSourceTests.cs diff --git a/src/ExcelReader.Core/Reader/Zip/ZipIndex.cs b/src/ExcelReader.Core/Reader/Zip/ZipIndex.cs index 060468bc..28e5d9a1 100644 --- a/src/ExcelReader.Core/Reader/Zip/ZipIndex.cs +++ b/src/ExcelReader.Core/Reader/Zip/ZipIndex.cs @@ -3,6 +3,7 @@ using System.IO.Compression; using System.Runtime.InteropServices; using ExcelReader.Core.Reader.Internal; +using ExcelReader.Core.Reader.Sources; namespace ExcelReader.Core.Reader.Zip { @@ -57,50 +58,192 @@ internal sealed class ZipIndex : IDisposable private static ReadOnlySpan EocdSignatureBytes => [0x50, 0x4B, 0x05, 0x06]; + private const int TailWindowSize = 65535 + EocdFixedSize + Zip64LocatorSize; + + private readonly ByteSource _source; private readonly ReadOnlyMemory _file; private readonly ReadOnlyMemory _directory; + private readonly byte[]? _directoryRented; private readonly ZipEntryRef[] _entries; - private bool _disposed; + private int _disposed; - private ZipIndex(ReadOnlyMemory file, ReadOnlyMemory directory, ZipEntryRef[] entries, int count) + private ZipIndex(ByteSource source, ReadOnlyMemory directory, byte[]? directoryRented, ZipEntryRef[] entries, int count) { - _file = file; + _source = source; + HasMemory = source.TryGetMemory(out _file); _directory = directory; + _directoryRented = directoryRented; _entries = entries; Count = count; } internal int Count { get; } + internal bool HasMemory { get; } + internal static ZipIndex Create(ReadOnlyMemory file, ExcelReaderOptions options) { - ReadOnlySpan span = file.Span; - long eocdOffset = FindEocd(span); - (long cdOffset, long cdSize, long declaredCount) = ReadEocdRecord(span, eocdOffset, out long zip64EocdOffset); - if (zip64EocdOffset >= 0) + return Create(ByteSource.FromMemory(file), options); + } + + internal static ZipIndex Create(ByteSource source, ExcelReaderOptions options) + { + byte[]? directoryRented = null; + try + { + long length = source.Length; + int tailSize = (int)Math.Min(length, TailWindowSize); + long cdOffset; + long cdSize; + long declaredCount; + long zip64EocdOffset; + if (source.TryGetMemory(out ReadOnlyMemory file)) + { + (cdOffset, cdSize, declaredCount, zip64EocdOffset) = ParseTail(file.Span[(file.Length - tailSize)..]); + } + else + { + byte[] tail = ArrayPool.Shared.Rent(Math.Max(1, tailSize)); + try + { + source.ReadExactly(length - tailSize, tail.AsSpan(0, tailSize)); + (cdOffset, cdSize, declaredCount, zip64EocdOffset) = ParseTail(tail.AsSpan(0, tailSize)); + } + finally + { + ArrayPool.Shared.Return(tail); + } + } + if (zip64EocdOffset >= 0) + { + ThrowIfZip64RecordOutOfRange(zip64EocdOffset, length); + Span record = stackalloc byte[Zip64EocdFixedSize]; + source.ReadExactly(zip64EocdOffset, record); + (cdOffset, cdSize, declaredCount) = ReadZip64Eocd(record, 0); + } + ThrowIfDirectoryOutOfRange(cdOffset, cdSize, length); + + ReadOnlyMemory directory; + if (source.TryGetMemory(out file)) + { + directory = file.Slice((int)cdOffset, (int)cdSize); + } + else + { + directoryRented = ArrayPool.Shared.Rent(Math.Max(1, (int)cdSize)); + source.ReadExactly(cdOffset, directoryRented.AsSpan(0, (int)cdSize)); + directory = directoryRented.AsMemory(0, (int)cdSize); + } + return Build(source, directory, directoryRented, declaredCount, options); + } + catch + { + if (directoryRented is not null) + { + ArrayPool.Shared.Return(directoryRented); + } + source.Dispose(); + throw; + } + } + + internal static async ValueTask CreateAsync(ByteSource source, ExcelReaderOptions options, CancellationToken ct) + { + if (source.TryGetMemory(out _)) + { + return Create(source, options); + } + byte[]? directoryRented = null; + try { - (cdOffset, cdSize, declaredCount) = ReadZip64Eocd(span, zip64EocdOffset); + long length = source.Length; + int tailSize = (int)Math.Min(length, TailWindowSize); + long cdOffset; + long cdSize; + long declaredCount; + long zip64EocdOffset; + byte[] tail = ArrayPool.Shared.Rent(Math.Max(1, tailSize)); + try + { + await source.ReadExactlyAsync(length - tailSize, tail.AsMemory(0, tailSize), ct).ConfigureAwait(false); + (cdOffset, cdSize, declaredCount, zip64EocdOffset) = ParseTail(tail.AsSpan(0, tailSize)); + } + finally + { + ArrayPool.Shared.Return(tail); + } + if (zip64EocdOffset >= 0) + { + ThrowIfZip64RecordOutOfRange(zip64EocdOffset, length); + byte[] record = ArrayPool.Shared.Rent(Zip64EocdFixedSize); + try + { + await source.ReadExactlyAsync(zip64EocdOffset, record.AsMemory(0, Zip64EocdFixedSize), ct).ConfigureAwait(false); + (cdOffset, cdSize, declaredCount) = ReadZip64Eocd(record.AsSpan(0, Zip64EocdFixedSize), 0); + } + finally + { + ArrayPool.Shared.Return(record); + } + } + ThrowIfDirectoryOutOfRange(cdOffset, cdSize, length); + directoryRented = ArrayPool.Shared.Rent(Math.Max(1, (int)cdSize)); + await source.ReadExactlyAsync(cdOffset, directoryRented.AsMemory(0, (int)cdSize), ct).ConfigureAwait(false); + return Build(source, directoryRented.AsMemory(0, (int)cdSize), directoryRented, declaredCount, options); } - if (cdOffset < 0 || cdSize < 0 || cdOffset > span.Length - cdSize) + catch + { + if (directoryRented is not null) + { + ArrayPool.Shared.Return(directoryRented); + } + source.Dispose(); + throw; + } + } + + private static (long CdOffset, long CdSize, long Count, long Zip64EocdOffset) ParseTail(ReadOnlySpan tail) + { + long eocdOffset = FindEocd(tail); + (long cdOffset, long cdSize, long declaredCount) = ReadEocdRecord(tail, eocdOffset, out long zip64EocdOffset); + return (cdOffset, cdSize, declaredCount, zip64EocdOffset); + } + + private static void ThrowIfZip64RecordOutOfRange(long offset, long length) + { + if (offset < 0 || offset > length - Zip64EocdFixedSize) + { + throw new InvalidDataException("The ZIP64 end of central directory record is out of range."); + } + } + + private static void ThrowIfDirectoryOutOfRange(long cdOffset, long cdSize, long length) + { + if (cdOffset < 0 || cdSize < 0 || cdOffset > length - cdSize) { throw new InvalidDataException("The ZIP central directory is out of range."); } - ReadOnlyMemory directory = file.Slice((int)cdOffset, (int)cdSize); + if (cdSize > Array.MaxLength) + { + throw new ExcelLimitExceededException("ArrayMaxLength", Array.MaxLength, cdSize); + } + } + private static ZipIndex Build(ByteSource source, ReadOnlyMemory directory, byte[]? directoryRented, long declaredCount, ExcelReaderOptions options) + { long maxHint = Math.Max(16, options.MaxZipEntries > 0 ? options.MaxZipEntries : 65_536); ZipEntryRef[] entries = ArrayPool.Shared.Rent((int)Math.Clamp(declaredCount, 16, maxHint)); - int count; try { - count = WalkCentralDirectory(directory.Span, ref entries, options); + int count = WalkCentralDirectory(directory.Span, ref entries, options); ThrowIfDuplicateEntryNames(directory, entries, count); + return new ZipIndex(source, directory, directoryRented, entries, count); } catch { ArrayPool.Shared.Return(entries); throw; } - return new ZipIndex(file, directory, entries, count); } private static void ThrowIfDuplicateEntryNames(ReadOnlyMemory directory, ZipEntryRef[] entries, int count) @@ -157,6 +300,110 @@ internal bool TryGetEntry(ReadOnlySpan utf8Name, out ZipEntryRef entry) } internal ZipPart OpenPart(in ZipEntryRef entry, DecompressedByteCounter counter, string entryLimitName = "", long entryLimit = 0) + { + ThrowIfPartTooLarge(entry, counter, entryLimitName, entryLimit); + long dataOffset = ResolveDataOffset(entry); + ThrowIfDataOutOfRange(dataOffset, entry.CompressedSize); + ZipPart part; + if (HasMemory) + { + ReadOnlyMemory compressed = CompressedSlice(dataOffset, entry.CompressedSize); + part = entry.Method switch + { + 0 => new ZipPart(compressed, rented: null), + 8 => InflateToPart(compressed, entry.UncompressedSize), + _ => throw new NotSupportedException($"Unsupported ZIP compression method: {entry.Method}."), + }; + } + else + { + ThrowIfMethodUnsupported(entry.Method); + part = ReadPart(dataOffset, entry); + } + counter.Add(entry.UncompressedSize); + return part; + } + + internal async ValueTask OpenPartAsync( + ZipEntryRef entry, DecompressedByteCounter counter, CancellationToken ct, string entryLimitName = "", long entryLimit = 0) + { + if (HasMemory) + { + return OpenPart(entry, counter, entryLimitName, entryLimit); + } + ThrowIfPartTooLarge(entry, counter, entryLimitName, entryLimit); + long dataOffset = await ResolveDataOffsetAsync(entry, ct).ConfigureAwait(false); + ThrowIfDataOutOfRange(dataOffset, entry.CompressedSize); + ThrowIfMethodUnsupported(entry.Method); + int size = PartLength(entry); + byte[] rented = ArrayPool.Shared.Rent(Math.Max(1, size)); + try + { + ByteSourceStream raw = new(_source, dataOffset, entry.CompressedSize); + Stream content = entry.Method == 0 ? raw : new DeflateStream(raw, CompressionMode.Decompress); + await using (content.ConfigureAwait(false)) + { + try + { + await content.ReadExactlyAsync(rented.AsMemory(0, size), ct).ConfigureAwait(false); + } + catch (EndOfStreamException ex) + { + throw new InvalidDataException("The ZIP entry produced less data than its declared uncompressed size.", ex); + } + if (await content.ReadAsync(new byte[1], ct).ConfigureAwait(false) != 0) + { + throw new InvalidDataException("The ZIP entry produced more data than its declared uncompressed size."); + } + } + } + catch + { + ArrayPool.Shared.Return(rented); + throw; + } + counter.Add(entry.UncompressedSize); + return new ZipPart(rented.AsMemory(0, size), rented); + } + + internal ValueTask OpenPartOrDefaultAsync(ReadOnlySpan utf8Name, DecompressedByteCounter counter, CancellationToken ct) + { + return TryGetEntry(utf8Name, out ZipEntryRef entry) ? OpenPartAsync(entry, counter, ct) : new ValueTask(default(ZipPart)); + } + + internal ZipPart OpenPartOrDefault(ReadOnlySpan utf8Name, DecompressedByteCounter counter) + { + return TryGetEntry(utf8Name, out ZipEntryRef entry) ? OpenPart(entry, counter) : default; + } + + internal LimitedReadStream OpenEntryStream( + in ZipEntryRef entry, DecompressedByteCounter counter, ExcelReaderOptions options, string entryLimitName = "", long entryLimit = 0) + { + long dataOffset = ResolveDataOffset(entry); + return WrapEntryStream(entry, dataOffset, counter, options, entryLimitName, entryLimit); + } + + internal async ValueTask OpenEntryStreamAsync( + ZipEntryRef entry, DecompressedByteCounter counter, ExcelReaderOptions options, CancellationToken ct, + string entryLimitName = "", long entryLimit = 0) + { + long dataOffset = HasMemory ? ResolveDataOffset(entry) : await ResolveDataOffsetAsync(entry, ct).ConfigureAwait(false); + return WrapEntryStream(entry, dataOffset, counter, options, entryLimitName, entryLimit); + } + + private LimitedReadStream WrapEntryStream( + in ZipEntryRef entry, long dataOffset, DecompressedByteCounter counter, ExcelReaderOptions options, string entryLimitName, long entryLimit) + { + ThrowIfDataOutOfRange(dataOffset, entry.CompressedSize); + ThrowIfMethodUnsupported(entry.Method); + Stream raw = HasMemory + ? ToReadableMemoryStream(CompressedSlice(dataOffset, entry.CompressedSize)) + : new ByteSourceStream(_source, dataOffset, entry.CompressedSize); + Stream opened = entry.Method == 0 ? raw : new DeflateStream(raw, CompressionMode.Decompress); + return WorkbookLookups.Wrap(opened, counter, options, entryLimitName, entryLimit, entry.UncompressedSize); + } + + private static void ThrowIfPartTooLarge(in ZipEntryRef entry, DecompressedByteCounter counter, string entryLimitName, long entryLimit) { LimitChecks.ThrowIfEntryLengthExceeds(entry.UncompressedSize, counter.Remaining, nameof(ExcelReaderOptions.MaxTotalDecompressedBytes)); if (entryLimit > 0) @@ -167,61 +414,161 @@ internal ZipPart OpenPart(in ZipEntryRef entry, DecompressedByteCounter counter, { throw new ExcelLimitExceededException("ArrayMaxLength", Array.MaxLength, entry.UncompressedSize); } + } - ReadOnlyMemory compressed = ResolveCompressedSlice(entry); - ZipPart part = entry.Method switch + private static void ThrowIfMethodUnsupported(ushort method) + { + if (method is not (0 or 8)) { - 0 => new ZipPart(compressed, rented: null), - 8 => InflateToPart(compressed, entry.UncompressedSize), - _ => throw new NotSupportedException($"Unsupported ZIP compression method: {entry.Method}."), - }; - counter.Add(entry.UncompressedSize); - return part; + throw new NotSupportedException($"Unsupported ZIP compression method: {method}."); + } } - internal ZipPart OpenPartOrDefault(ReadOnlySpan utf8Name, DecompressedByteCounter counter) + private void ThrowIfDataOutOfRange(long dataOffset, long compressedSize) { - return TryGetEntry(utf8Name, out ZipEntryRef entry) ? OpenPart(entry, counter) : default; + long length = _source.Length; + if (dataOffset < 0 || dataOffset > length || compressedSize < 0 || compressedSize > length - dataOffset) + { + throw new InvalidDataException("The ZIP entry data runs past the end of the file."); + } } - internal Stream OpenEntryStream(in ZipEntryRef entry, DecompressedByteCounter counter, ExcelReaderOptions options, string entryLimitName = "", long entryLimit = 0) + private ReadOnlyMemory CompressedSlice(long dataOffset, long compressedSize) { - ReadOnlyMemory compressed = ResolveCompressedSlice(entry); - Stream opened = entry.Method switch + if (compressedSize > Array.MaxLength) { - 0 => ToReadableMemoryStream(compressed), - 8 => new DeflateStream(ToReadableMemoryStream(compressed), CompressionMode.Decompress), - _ => throw new NotSupportedException($"Unsupported ZIP compression method: {entry.Method}."), - }; - return WorkbookLookups.Wrap(opened, counter, options, entryLimitName, entryLimit, entry.UncompressedSize); + throw new ExcelLimitExceededException("ArrayMaxLength", Array.MaxLength, compressedSize); + } + return _file.Slice((int)dataOffset, (int)compressedSize); } - private ReadOnlyMemory ResolveCompressedSlice(in ZipEntryRef entry) + private static int PartLength(in ZipEntryRef entry) { - long dataOffset = ResolveDataOffset(entry); - long fileLength = _file.Length; - if (dataOffset < 0 || dataOffset > fileLength || entry.CompressedSize < 0 || entry.CompressedSize > fileLength - dataOffset) + long length = entry.Method == 0 ? entry.CompressedSize : entry.UncompressedSize; + if (length > Array.MaxLength) { - throw new InvalidDataException("The ZIP entry data runs past the end of the file."); + throw new ExcelLimitExceededException("ArrayMaxLength", Array.MaxLength, length); + } + return (int)length; + } + + private ZipPart ReadPart(long dataOffset, in ZipEntryRef entry) + { + int size = PartLength(entry); + byte[] rented = ArrayPool.Shared.Rent(Math.Max(1, size)); + try + { + ByteSourceStream raw = new(_source, dataOffset, entry.CompressedSize); + using Stream content = entry.Method == 0 ? raw : new DeflateStream(raw, CompressionMode.Decompress); + try + { + content.ReadExactly(rented.AsSpan(0, size)); + } + catch (EndOfStreamException ex) + { + throw new InvalidDataException("The ZIP entry produced less data than its declared uncompressed size.", ex); + } + if (content.ReadByte() != -1) + { + throw new InvalidDataException("The ZIP entry produced more data than its declared uncompressed size."); + } + return new ZipPart(rented.AsMemory(0, size), rented); } - if (entry.CompressedSize > Array.MaxLength) + catch { - throw new ExcelLimitExceededException("ArrayMaxLength", Array.MaxLength, entry.CompressedSize); + ArrayPool.Shared.Return(rented); + throw; + } + } + + private long ResolveDataOffset(in ZipEntryRef entry) + { + ThrowIfLocalHeaderOutOfRange(entry.LocalHeaderOffset); + Span header = stackalloc byte[LocalHeaderFixedSize]; + _source.ReadExactly(entry.LocalHeaderOffset, header); + long nameStart = ParseLocalHeader(header, entry, out int nameLength, out int extraLength); + byte[] name = ArrayPool.Shared.Rent(Math.Max(1, nameLength)); + try + { + _source.ReadExactly(nameStart, name.AsSpan(0, nameLength)); + ThrowIfNameMismatch(name.AsSpan(0, nameLength), entry); + } + finally + { + ArrayPool.Shared.Return(name); + } + return nameStart + nameLength + extraLength; + } + + private async ValueTask ResolveDataOffsetAsync(ZipEntryRef entry, CancellationToken ct) + { + ThrowIfLocalHeaderOutOfRange(entry.LocalHeaderOffset); + byte[] buffer = ArrayPool.Shared.Rent(LocalHeaderFixedSize + ushort.MaxValue); + try + { + await _source.ReadExactlyAsync(entry.LocalHeaderOffset, buffer.AsMemory(0, LocalHeaderFixedSize), ct).ConfigureAwait(false); + long nameStart = ParseLocalHeader(buffer.AsSpan(0, LocalHeaderFixedSize), entry, out int nameLength, out int extraLength); + await _source.ReadExactlyAsync(nameStart, buffer.AsMemory(LocalHeaderFixedSize, nameLength), ct).ConfigureAwait(false); + ThrowIfNameMismatch(buffer.AsSpan(LocalHeaderFixedSize, nameLength), entry); + return nameStart + nameLength + extraLength; + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + private void ThrowIfLocalHeaderOutOfRange(long headerOffset) + { + if (headerOffset < 0 || headerOffset > _source.Length - LocalHeaderFixedSize) + { + throw new InvalidDataException("The ZIP local file header is out of range."); + } + } + + private long ParseLocalHeader(ReadOnlySpan header, in ZipEntryRef entry, out int nameLength, out int extraLength) + { + if (BinaryPrimitives.ReadInt32LittleEndian(header) != LocalFileHeaderSignature) + { + throw new InvalidDataException("Invalid ZIP local file header signature."); + } + nameLength = BinaryPrimitives.ReadUInt16LittleEndian(header[26..]); + extraLength = BinaryPrimitives.ReadUInt16LittleEndian(header[28..]); + long nameStart = entry.LocalHeaderOffset + LocalHeaderFixedSize; + if (nameLength > _source.Length - nameStart) + { + throw new InvalidDataException("The ZIP local file header name runs past the end of the file."); + } + if (nameLength != entry.NameLength) + { + throw new InvalidDataException("The ZIP local file header name does not match the central directory."); + } + return nameStart; + } + + private void ThrowIfNameMismatch(ReadOnlySpan localName, in ZipEntryRef entry) + { + if (!localName.SequenceEqual(_directory.Span.Slice(entry.NameStart, entry.NameLength))) + { + throw new InvalidDataException("The ZIP local file header name does not match the central directory."); } - return _file.Slice((int)dataOffset, (int)entry.CompressedSize); } public void Dispose() { - if (_disposed) + if (Interlocked.Exchange(ref _disposed, 1) != 0) { return; } - _disposed = true; if (_entries.Length > 0) { ArrayPool.Shared.Return(_entries); } + if (_directoryRented is not null) + { + ArrayPool.Shared.Return(_directoryRented); + } + _source.Dispose(); } private static long FindEocd(ReadOnlySpan span) @@ -449,34 +796,6 @@ private static void EnsureCapacity(ref ZipEntryRef[] entries, int count) entries = bigger; } - private long ResolveDataOffset(in ZipEntryRef entry) - { - ReadOnlySpan fileSpan = _file.Span; - long headerOffset = entry.LocalHeaderOffset; - if (headerOffset < 0 || headerOffset > fileSpan.Length - LocalHeaderFixedSize) - { - throw new InvalidDataException("The ZIP local file header is out of range."); - } - ReadOnlySpan header = fileSpan.Slice((int)headerOffset, LocalHeaderFixedSize); - if (BinaryPrimitives.ReadInt32LittleEndian(header) != LocalFileHeaderSignature) - { - throw new InvalidDataException("Invalid ZIP local file header signature."); - } - ushort nameLength = BinaryPrimitives.ReadUInt16LittleEndian(header[26..]); - ushort extraLength = BinaryPrimitives.ReadUInt16LittleEndian(header[28..]); - long nameStart = headerOffset + LocalHeaderFixedSize; - if (nameLength > fileSpan.Length - nameStart) - { - throw new InvalidDataException("The ZIP local file header name runs past the end of the file."); - } - if (nameLength != entry.NameLength || - !fileSpan.Slice((int)nameStart, nameLength).SequenceEqual(_directory.Span.Slice(entry.NameStart, entry.NameLength))) - { - throw new InvalidDataException("The ZIP local file header name does not match the central directory."); - } - return nameStart + nameLength + extraLength; - } - private static ZipPart InflateToPart(ReadOnlyMemory compressed, long uncompressedSize) { int size = checked((int)uncompressedSize); diff --git a/tests/ExcelReader.Tests/Reader/Zip/ZipIndexSourceTests.cs b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexSourceTests.cs new file mode 100644 index 00000000..3574f9a6 --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexSourceTests.cs @@ -0,0 +1,306 @@ +using System.IO.Compression; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Sources; +using ExcelReader.Core.Reader.Zip; + +namespace ExcelReader.Tests.Reader.Zip +{ + public sealed class ZipIndexSourceTests : IDisposable + { + public enum Kind + { + Memory, + OpaqueMemoryStream, + Trickle, + File, + } + + private static readonly string[] Names = ["deflated.txt", "stored.bin", "empty"]; + private static readonly int[] Sizes = [300_000, 70_000, 0]; + + private readonly List _paths = []; + + public void Dispose() + { + foreach (string path in _paths) + { + File.Delete(path); + } + } + + public static TheoryData KindsAndDescriptors + { + get + { + TheoryData data = []; + foreach (Kind kind in Enum.GetValues()) + { + data.Add(kind, false); + data.Add(kind, true); + } + return data; + } + } + + public static TheoryData Kinds => [Kind.Memory, Kind.OpaqueMemoryStream, Kind.Trickle, Kind.File]; + + private static byte[] Content(int entry, int scale = 1) + { + byte[] bytes = new byte[Sizes[entry] / scale]; + for (int i = 0; i < bytes.Length; i++) + { + bytes[i] = (byte)((i % 251) ^ (entry * 37)); + } + return bytes; + } + + private sealed class ForwardOnlyStream(Stream inner) : Stream + { + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => throw new NotSupportedException(); + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() + { + inner.Flush(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + throw new NotSupportedException(); + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotSupportedException(); + } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + inner.Write(buffer, offset, count); + } + } + + private static byte[] BuildZip(bool dataDescriptors, int scale = 1) + { + using MemoryStream buffer = new(); + using ForwardOnlyStream forward = new(buffer); + using (ZipArchive zip = new(dataDescriptors ? forward : buffer, ZipArchiveMode.Create, leaveOpen: true)) + { + for (int i = 0; i < Names.Length; i++) + { + ZipArchiveEntry entry = zip.CreateEntry(Names[i], i == 1 ? CompressionLevel.NoCompression : CompressionLevel.Optimal); + using Stream stream = entry.Open(); + stream.Write(Content(i, scale)); + } + } + return buffer.ToArray(); + } + + private ByteSource Open(Kind kind, byte[] zipBytes) + { + switch (kind) + { + case Kind.Memory: + return ByteSource.FromMemory(zipBytes); + case Kind.OpaqueMemoryStream: + return ByteSource.FromStream(new MemoryStream(zipBytes, writable: false), leaveOpen: false); + case Kind.Trickle: + return ByteSource.FromStream(new TrickleStream(zipBytes), leaveOpen: false); + default: + string path = Path.Combine(Path.GetTempPath(), "excelreader-zipindex-" + Guid.NewGuid().ToString("N") + ".zip"); + File.WriteAllBytes(path, zipBytes); + _paths.Add(path); + return ByteSource.OpenFile(path, asynchronous: false); + } + } + + private static DecompressedByteCounter Counter() + { + return new DecompressedByteCounter(ExcelReaderOptions.Default.MaxTotalDecompressedBytes); + } + + [Theory] + [MemberData(nameof(KindsAndDescriptors))] + public void Parts_Hold_The_Entry_Bytes(Kind kind, bool dataDescriptors) + { + using ZipIndex index = ZipIndex.Create(Open(kind, BuildZip(dataDescriptors)), ExcelReaderOptions.Default); + Assert.Equal(kind == Kind.Memory, index.HasMemory); + for (int i = 0; i < Names.Length; i++) + { + Assert.True(index.TryGetEntry(System.Text.Encoding.UTF8.GetBytes(Names[i]), out ZipEntryRef entry)); + using ZipPart part = index.OpenPart(entry, Counter()); + Assert.True(part.Memory.Span.SequenceEqual(Content(i))); + } + } + + [Theory] + [MemberData(nameof(KindsAndDescriptors))] + public void Entry_Streams_Yield_The_Entry_Bytes(Kind kind, bool dataDescriptors) + { + using ZipIndex index = ZipIndex.Create(Open(kind, BuildZip(dataDescriptors)), ExcelReaderOptions.Default); + for (int i = 0; i < Names.Length; i++) + { + Assert.True(index.TryGetEntry(System.Text.Encoding.UTF8.GetBytes(Names[i]), out ZipEntryRef entry)); + using Stream stream = index.OpenEntryStream(entry, Counter(), ExcelReaderOptions.Default); + using MemoryStream copy = new(); + stream.CopyTo(copy); + Assert.True(copy.ToArray().AsSpan().SequenceEqual(Content(i))); + } + } + + [Theory] + [MemberData(nameof(KindsAndDescriptors))] + public async Task Async_Open_Matches_Sync_Open(Kind kind, bool dataDescriptors) + { + CancellationToken ct = TestContext.Current.CancellationToken; + using ZipIndex index = await ZipIndex.CreateAsync(Open(kind, BuildZip(dataDescriptors)), ExcelReaderOptions.Default, ct); + for (int i = 0; i < Names.Length; i++) + { + byte[] name = System.Text.Encoding.UTF8.GetBytes(Names[i]); + using ZipPart part = await index.OpenPartOrDefaultAsync(name, Counter(), ct); + Assert.True(part.Memory.Span.SequenceEqual(Content(i))); + + Assert.True(index.TryGetEntry(name, out ZipEntryRef entry)); + await using Stream stream = await index.OpenEntryStreamAsync(entry, Counter(), ExcelReaderOptions.Default, ct); + using MemoryStream copy = new(); + await stream.CopyToAsync(copy, ct); + Assert.True(copy.ToArray().AsSpan().SequenceEqual(Content(i))); + } + } + + [Theory] + [MemberData(nameof(Kinds))] + public void Entries_Open_Concurrently(Kind kind) + { + using ZipIndex index = ZipIndex.Create(Open(kind, BuildZip(dataDescriptors: false, scale: kind == Kind.Trickle ? 20 : 1)), ExcelReaderOptions.Default); + int scale = kind == Kind.Trickle ? 20 : 1; + Parallel.For(0, 24, worker => + { + int which = worker % 2; + Assert.True(index.TryGetEntry(System.Text.Encoding.UTF8.GetBytes(Names[which]), out ZipEntryRef entry)); + using Stream stream = index.OpenEntryStream(entry, Counter(), ExcelReaderOptions.Default); + using MemoryStream copy = new(); + stream.CopyTo(copy); + Assert.True(copy.ToArray().AsSpan().SequenceEqual(Content(which, scale))); + }); + } + + [Theory] + [MemberData(nameof(Kinds))] + public void A_File_Cut_Short_Is_Invalid_Data(Kind kind) + { + byte[] zipBytes = BuildZip(dataDescriptors: false, scale: 20); + Assert.Throws( + () => ZipIndex.Create(Open(kind, zipBytes[..(zipBytes.Length / 2)]), ExcelReaderOptions.Default)); + Assert.Throws( + () => ZipIndex.Create(Open(kind, []), ExcelReaderOptions.Default)); + } + + [Fact] + public void A_Failed_Create_Disposes_The_Source() + { + TrickleStream stream = new(new byte[100]); + Assert.Throws( + () => ZipIndex.Create(ByteSource.FromStream(stream, leaveOpen: false), ExcelReaderOptions.Default)); + Assert.False(stream.CanRead); + } + + private static string Outcome(Func open) + { + try + { + using ZipIndex index = ZipIndex.Create(open(), ExcelReaderOptions.Default); + System.Text.StringBuilder text = new(); + foreach (string name in Names) + { + if (!index.TryGetEntry(System.Text.Encoding.UTF8.GetBytes(name), out ZipEntryRef entry)) + { + text.Append("missing;"); + continue; + } + try + { + using ZipPart part = index.OpenPart(entry, Counter()); + text.Append(Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(part.Memory.Span))).Append(';'); + } + catch (Exception ex) when (ex is InvalidDataException or NotSupportedException or ExcelLimitExceededException) + { + text.Append(ex.GetType().Name).Append(';'); + } + } + return text.ToString(); + } + catch (Exception ex) when (ex is InvalidDataException or NotSupportedException or ExcelLimitExceededException) + { + return ex.GetType().Name; + } + } + + [Fact] + public void Mutated_Bytes_Give_The_Same_Outcome_From_Memory_And_From_A_Stream() + { + byte[] pristine = BuildZip(dataDescriptors: false, scale: 1000); + uint state = 2024; + for (int i = 0; i < 3000; i++) + { + byte[] mutated = (byte[])pristine.Clone(); + state = (state * 1664525) + 1013904223; + int position = (int)(state % (uint)mutated.Length); + state = (state * 1664525) + 1013904223; + mutated[position] = (byte)(state >> 24); + + string fromMemory = Outcome(() => ByteSource.FromMemory(mutated)); + string fromStream = Outcome(() => ByteSource.FromStream(new MemoryStream(mutated, writable: false), leaveOpen: false)); + Assert.True(string.Equals(fromMemory, fromStream, StringComparison.Ordinal),$"mutation {i} at byte {position}: memory={fromMemory} stream={fromStream}"); + } + } + + [Fact] + public void Test_Workbooks_Match_ZipArchive_Entry_For_Entry() + { + string dataDirectory = Path.Combine(AppContext.BaseDirectory, "data"); + int checkedFiles = 0; + foreach (string path in Directory.EnumerateFiles(dataDirectory, "*.xls?", SearchOption.AllDirectories)) + { + byte[] bytes = File.ReadAllBytes(path); + if (bytes.Length < 4 || bytes[0] != (byte)'P' || bytes[1] != (byte)'K') + { + continue; + } + using ZipArchive expected = new(new MemoryStream(bytes, writable: false), ZipArchiveMode.Read); + using ZipIndex index = ZipIndex.Create(ByteSource.OpenFile(path, asynchronous: false), ExcelReaderOptions.Default); + foreach (ZipArchiveEntry expectedEntry in expected.Entries) + { + if (expectedEntry.FullName.EndsWith('/')) + { + continue; + } + Assert.True(index.TryGetEntry(System.Text.Encoding.UTF8.GetBytes(expectedEntry.FullName), out ZipEntryRef entry), expectedEntry.FullName); + using Stream expectedStream = expectedEntry.Open(); + using MemoryStream expectedBytes = new(); + expectedStream.CopyTo(expectedBytes); + using ZipPart part = index.OpenPart(entry, Counter()); + Assert.True(part.Memory.Span.SequenceEqual(expectedBytes.ToArray()), path + "!" + expectedEntry.FullName); + } + checkedFiles++; + } + Assert.True(checkedFiles > 0, "no ZIP-based workbook found under " + dataDirectory); + } + } +} From 5f794085d9a2d3bad10102b47327f722a09029c5 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Tue, 6 Oct 2026 01:19:44 -0300 Subject: [PATCH 06/28] fix(core): reject stored ZIP entries whose sizes disagree Co-Authored-By: Claude Opus 5.5 --- src/ExcelReader.Core/Reader/Zip/ZipIndex.cs | 50 ++++++------------- .../Reader/Zip/ZipIndexSourceTests.cs | 24 +++++++++ .../Reader/Zip/ZipIndexTests.cs | 6 +-- 3 files changed, 43 insertions(+), 37 deletions(-) diff --git a/src/ExcelReader.Core/Reader/Zip/ZipIndex.cs b/src/ExcelReader.Core/Reader/Zip/ZipIndex.cs index 28e5d9a1..ec9ae6c5 100644 --- a/src/ExcelReader.Core/Reader/Zip/ZipIndex.cs +++ b/src/ExcelReader.Core/Reader/Zip/ZipIndex.cs @@ -303,21 +303,15 @@ internal ZipPart OpenPart(in ZipEntryRef entry, DecompressedByteCounter counter, { ThrowIfPartTooLarge(entry, counter, entryLimitName, entryLimit); long dataOffset = ResolveDataOffset(entry); - ThrowIfDataOutOfRange(dataOffset, entry.CompressedSize); + ThrowIfEntryDataInvalid(dataOffset, entry); ZipPart part; if (HasMemory) { ReadOnlyMemory compressed = CompressedSlice(dataOffset, entry.CompressedSize); - part = entry.Method switch - { - 0 => new ZipPart(compressed, rented: null), - 8 => InflateToPart(compressed, entry.UncompressedSize), - _ => throw new NotSupportedException($"Unsupported ZIP compression method: {entry.Method}."), - }; + part = entry.Method == 0 ? new ZipPart(compressed, rented: null) : InflateToPart(compressed, entry.UncompressedSize); } else { - ThrowIfMethodUnsupported(entry.Method); part = ReadPart(dataOffset, entry); } counter.Add(entry.UncompressedSize); @@ -333,9 +327,8 @@ internal async ValueTask OpenPartAsync( } ThrowIfPartTooLarge(entry, counter, entryLimitName, entryLimit); long dataOffset = await ResolveDataOffsetAsync(entry, ct).ConfigureAwait(false); - ThrowIfDataOutOfRange(dataOffset, entry.CompressedSize); - ThrowIfMethodUnsupported(entry.Method); - int size = PartLength(entry); + ThrowIfEntryDataInvalid(dataOffset, entry); + int size = (int)entry.UncompressedSize; byte[] rented = ArrayPool.Shared.Rent(Math.Max(1, size)); try { @@ -394,8 +387,7 @@ internal async ValueTask OpenEntryStreamAsync( private LimitedReadStream WrapEntryStream( in ZipEntryRef entry, long dataOffset, DecompressedByteCounter counter, ExcelReaderOptions options, string entryLimitName, long entryLimit) { - ThrowIfDataOutOfRange(dataOffset, entry.CompressedSize); - ThrowIfMethodUnsupported(entry.Method); + ThrowIfEntryDataInvalid(dataOffset, entry); Stream raw = HasMemory ? ToReadableMemoryStream(CompressedSlice(dataOffset, entry.CompressedSize)) : new ByteSourceStream(_source, dataOffset, entry.CompressedSize); @@ -416,21 +408,21 @@ private static void ThrowIfPartTooLarge(in ZipEntryRef entry, DecompressedByteCo } } - private static void ThrowIfMethodUnsupported(ushort method) - { - if (method is not (0 or 8)) - { - throw new NotSupportedException($"Unsupported ZIP compression method: {method}."); - } - } - - private void ThrowIfDataOutOfRange(long dataOffset, long compressedSize) + private void ThrowIfEntryDataInvalid(long dataOffset, in ZipEntryRef entry) { long length = _source.Length; - if (dataOffset < 0 || dataOffset > length || compressedSize < 0 || compressedSize > length - dataOffset) + if (dataOffset < 0 || dataOffset > length || entry.CompressedSize < 0 || entry.CompressedSize > length - dataOffset) { throw new InvalidDataException("The ZIP entry data runs past the end of the file."); } + if (entry.Method is not (0 or 8)) + { + throw new NotSupportedException($"Unsupported ZIP compression method: {entry.Method}."); + } + if (entry.Method == 0 && entry.CompressedSize != entry.UncompressedSize) + { + throw new InvalidDataException("The stored ZIP entry's compressed and uncompressed sizes differ."); + } } private ReadOnlyMemory CompressedSlice(long dataOffset, long compressedSize) @@ -442,19 +434,9 @@ private ReadOnlyMemory CompressedSlice(long dataOffset, long compressedSiz return _file.Slice((int)dataOffset, (int)compressedSize); } - private static int PartLength(in ZipEntryRef entry) - { - long length = entry.Method == 0 ? entry.CompressedSize : entry.UncompressedSize; - if (length > Array.MaxLength) - { - throw new ExcelLimitExceededException("ArrayMaxLength", Array.MaxLength, length); - } - return (int)length; - } - private ZipPart ReadPart(long dataOffset, in ZipEntryRef entry) { - int size = PartLength(entry); + int size = (int)entry.UncompressedSize; byte[] rented = ArrayPool.Shared.Rent(Math.Max(1, size)); try { diff --git a/tests/ExcelReader.Tests/Reader/Zip/ZipIndexSourceTests.cs b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexSourceTests.cs index 3574f9a6..ef6bf933 100644 --- a/tests/ExcelReader.Tests/Reader/Zip/ZipIndexSourceTests.cs +++ b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexSourceTests.cs @@ -212,6 +212,30 @@ public void A_File_Cut_Short_Is_Invalid_Data(Kind kind) () => ZipIndex.Create(Open(kind, []), ExcelReaderOptions.Default)); } + [Theory] + [MemberData(nameof(Kinds))] + public void A_Stored_Entry_Whose_Sizes_Disagree_Is_Invalid_Data(Kind kind) + { + byte[] zipBytes = BuildZip(dataDescriptors: false, scale: 20); + byte[] name = System.Text.Encoding.UTF8.GetBytes(Names[1]); + int record = -1; + for (int i = 0; i + 46 + name.Length <= zipBytes.Length; i++) + { + if (zipBytes.AsSpan(i, 4).SequenceEqual([0x50, 0x4B, 0x01, 0x02]) && zipBytes.AsSpan(i + 46, name.Length).SequenceEqual(name)) + { + record = i; + break; + } + } + Assert.True(record >= 0, "central directory record not found"); + zipBytes.AsSpan(record + 24, 4).Clear(); + + using ZipIndex index = ZipIndex.Create(Open(kind, zipBytes), ExcelReaderOptions.Default); + Assert.True(index.TryGetEntry(name, out ZipEntryRef entry)); + Assert.Throws(() => index.OpenPart(entry, Counter()).Dispose()); + Assert.Throws(() => index.OpenEntryStream(entry, Counter(), ExcelReaderOptions.Default).Dispose()); + } + [Fact] public void A_Failed_Create_Disposes_The_Source() { diff --git a/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs index e1e48226..3d2dea3c 100644 --- a/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs +++ b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs @@ -390,7 +390,7 @@ public void CompressedSizeEndingExactlyAtTheFileEndIsAcceptedAndOneMoreIsRejecte Zip64Layout probe = BuildStoredZip64(Zip64Payload); long exact = probe.Bytes.Length - probe.DataOffset; - Zip64Layout fits = BuildStoredZip64(Zip64Payload, compressedSize: exact); + Zip64Layout fits = BuildStoredZip64(Zip64Payload, compressedSize: exact, uncompressedSize: exact); using (ZipIndex index = ZipIndex.Create(fits.Bytes, ExcelReaderOptions.Default)) { Assert.True(index.TryGetEntry("a.txt"u8, out ZipEntryRef entry)); @@ -507,7 +507,7 @@ public void LocalExtraLengthPushingDataPastTheFileEndThrowsInvalidDataOnOpen() private readonly record struct Zip64Layout(byte[] Bytes, int DataOffset, int Zip64EocdOffset, int LocatorOffset); - private static Zip64Layout BuildStoredZip64(byte[] payload, long? localOffset = null, long? compressedSize = null) + private static Zip64Layout BuildStoredZip64(byte[] payload, long? localOffset = null, long? compressedSize = null, long? uncompressedSize = null) { byte[] name = "a.txt"u8.ToArray(); uint crc = Crc32(payload); @@ -544,7 +544,7 @@ private static Zip64Layout BuildStoredZip64(byte[] payload, long? localOffset = BinaryPrimitives.WriteUInt32LittleEndian(cd[42..], Zip64SentinelU32); name.CopyTo(cd[CentralDirectoryFixedSize..]); WriteZip64Extra(cd[(CentralDirectoryFixedSize + name.Length)..], - payload.Length, compressedSize ?? payload.Length, localOffset ?? 0); + uncompressedSize ?? payload.Length, compressedSize ?? payload.Length, localOffset ?? 0); Span z64 = s[zip64EocdOffset..]; BinaryPrimitives.WriteInt32LittleEndian(z64, 0x06064b50); From 9492ec3fa350f8b61cd8660ccd4638cfbcf19551 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Tue, 6 Oct 2026 01:26:55 -0300 Subject: [PATCH 07/28] refactor(core): read XLSX from streams and files through ZipIndex Co-Authored-By: Claude Sonnet 5.5 --- src/ExcelReader.Core/Reader/Excel.cs | 25 ++- .../Reader/Xlsx/XlsxReader.Enumerator.cs | 12 +- .../Reader/Xlsx/XlsxReader.Loading.cs | 31 ++- .../Reader/Xlsx/XlsxReader.Memory.cs | 50 ----- .../Reader/Xlsx/XlsxReader.cs | 135 ++++-------- .../Reader/SourceParityTests.cs | 205 ++++++++++++++++++ 6 files changed, 298 insertions(+), 160 deletions(-) create mode 100644 tests/ExcelReader.Tests/Reader/SourceParityTests.cs diff --git a/src/ExcelReader.Core/Reader/Excel.cs b/src/ExcelReader.Core/Reader/Excel.cs index bbca4065..a82116e7 100644 --- a/src/ExcelReader.Core/Reader/Excel.cs +++ b/src/ExcelReader.Core/Reader/Excel.cs @@ -377,7 +377,7 @@ public static IExcelRowReader Open(ReadOnlyMemory data, ExcelReaderOptions { ExcelFileFormat.Xls => new XlsReader(data, effective), ExcelFileFormat.Xlsb => XlsbReader.CreateFromMemory(memZip!, effective), - ExcelFileFormat.Xlsx => XlsxReader.CreateFromMemory(memZip!, effective), + ExcelFileFormat.Xlsx => XlsxReader.CreateFromIndex(memZip!, effective), _ => throw new System.Diagnostics.UnreachableException(), }; } @@ -393,7 +393,7 @@ private static IExcelRowReader OpenFromPlainMemory(ReadOnlyMemory plain, E return format switch { ExcelFileFormat.Xlsb => XlsbReader.CreateFromMemory(memZip!, options), - ExcelFileFormat.Xlsx => XlsxReader.CreateFromMemory(memZip!, options), + ExcelFileFormat.Xlsx => XlsxReader.CreateFromIndex(memZip!, options), _ => throw new System.Diagnostics.UnreachableException(), }; } @@ -690,7 +690,7 @@ private static IExcelRowReader OpenSeekable(Stream stream, bool leaveOpen, Excel { ExcelFileFormat.Xls => new XlsReader(stream, leaveOpen, options), ExcelFileFormat.Xlsb => new XlsbReader(stream, leaveOpen, zip!, options), - ExcelFileFormat.Xlsx => new XlsxReader(stream, leaveOpen, zip!, options), + ExcelFileFormat.Xlsx => ReopenXlsx(stream, leaveOpen, zip!, options), _ => throw new System.Diagnostics.UnreachableException(), }; } @@ -705,7 +705,7 @@ private static IExcelRowReader OpenDecryptedZip(Stream decrypted, ExcelReaderOpt return zipFormat switch { ExcelFileFormat.Xlsb => new XlsbReader(decrypted, leaveOpen: false, zip, options), - ExcelFileFormat.Xlsx => new XlsxReader(decrypted, leaveOpen: false, zip, options), + ExcelFileFormat.Xlsx => ReopenXlsx(decrypted, leaveOpen: false, zip, options), _ => throw new System.Diagnostics.UnreachableException(), }; } @@ -749,7 +749,7 @@ private static async ValueTask OpenSeekableAsync(Stream stream, { ExcelFileFormat.Xls => await XlsReader.CreateAsync(stream, leaveOpen, options, ct).ConfigureAwait(false), ExcelFileFormat.Xlsb => await XlsbReader.CreateFromOpenZipAsync(stream, leaveOpen, zip!, options, ct).ConfigureAwait(false), - ExcelFileFormat.Xlsx => await XlsxReader.CreateFromOpenZipAsync(stream, leaveOpen, zip!, options, ct).ConfigureAwait(false), + ExcelFileFormat.Xlsx => await ReopenXlsxAsync(stream, leaveOpen, zip!, options, ct).ConfigureAwait(false), _ => throw new System.Diagnostics.UnreachableException(), }; } @@ -764,7 +764,7 @@ private static async ValueTask OpenDecryptedZipAsync(Stream dec return zipFormat switch { ExcelFileFormat.Xlsb => await XlsbReader.CreateFromOpenZipAsync(decrypted, leaveOpen: false, zip, options, ct).ConfigureAwait(false), - ExcelFileFormat.Xlsx => await XlsxReader.CreateFromOpenZipAsync(decrypted, leaveOpen: false, zip, options, ct).ConfigureAwait(false), + ExcelFileFormat.Xlsx => await ReopenXlsxAsync(decrypted, leaveOpen: false, zip, options, ct).ConfigureAwait(false), _ => throw new System.Diagnostics.UnreachableException(), }; } @@ -816,6 +816,19 @@ private static bool TryClassifyHeader(ReadOnlySpan sig, out ExcelFileForma return true; } + private static XlsxReader ReopenXlsx(Stream stream, bool leaveOpen, ZipArchive peeked, ExcelReaderOptions? options) + { + peeked.Dispose(); + return new XlsxReader(stream, leaveOpen, options); + } + + private static async ValueTask ReopenXlsxAsync( + Stream stream, bool leaveOpen, ZipArchive peeked, ExcelReaderOptions? options, CancellationToken ct) + { + await peeked.DisposeAsync().ConfigureAwait(false); + return await XlsxReader.CreateAsync(stream, leaveOpen, options, ct).ConfigureAwait(false); + } + private static ExcelFileFormat ClassifyZipStream(Stream stream, long start, out ZipArchive zip) { var zipPeek = new ZipArchive(stream, ZipArchiveMode.Read, leaveOpen: true); diff --git a/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Enumerator.cs b/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Enumerator.cs index 05a39cc5..b5db839d 100644 --- a/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Enumerator.cs +++ b/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Enumerator.cs @@ -1,9 +1,9 @@ using System.Buffers.Text; using System.Diagnostics.CodeAnalysis; using System.Globalization; -using System.IO.Compression; using System.Runtime.CompilerServices; using ExcelReader.Core.Reader.Internal; +using ExcelReader.Core.Reader.Zip; using ExcelReader.Core.Writer.Internal; namespace ExcelReader.Core.Reader.Xlsx @@ -21,7 +21,7 @@ public sealed class Enumerator : PooledStreamRowEnumerator, IExcelRowEnumerator private readonly bool[] _styleIsDate; private int[] _sharedOffsets; private readonly Utf8StringCache? _contentCache; - private readonly ZipArchiveEntry? _entry; + private readonly ZipEntryRef _entry; private int _nextCol; private NsTokens? _ns; @@ -40,8 +40,8 @@ internal Enumerator(XlsxReader reader, Stream sheet, long entryLength = 0, Cance _contentCache = reader._options.InternStrings ? new Utf8StringCache() : null; } - internal Enumerator(XlsxReader reader, ZipArchiveEntry entry, CancellationToken ct) - : base(reader._options.MaxCellBytes, nameof(ExcelReaderOptions.MaxCellBytes), WorkbookLookups.InitialBufferCapacity(entry.Length), ct) + internal Enumerator(XlsxReader reader, ZipEntryRef entry, CancellationToken ct) + : base(reader._options.MaxCellBytes, nameof(ExcelReaderOptions.MaxCellBytes), WorkbookLookups.InitialBufferCapacity(entry.UncompressedSize), ct) { _reader = reader; _styleIsDate = reader._styleIsDate; @@ -54,14 +54,14 @@ private protected override Stream OpenSource() { _reader.EnsureSharedLoaded(); _sharedOffsets = _reader._sharedOffsets; - return WorkbookLookups.OpenEntryStream(_entry!, _reader._decompressedBytes, _reader._options); + return _reader._zip.OpenEntryStream(_entry, _reader._decompressedBytes, _reader._options); } private protected override async ValueTask OpenSourceAsync() { await _reader.EnsureSharedLoadedAsync(_ct).ConfigureAwait(false); _sharedOffsets = _reader._sharedOffsets; - return await WorkbookLookups.OpenEntryStreamAsync(_entry!, _reader._decompressedBytes, _reader._options, _ct).ConfigureAwait(false); + return await _reader._zip.OpenEntryStreamAsync(_entry, _reader._decompressedBytes, _reader._options, _ct).ConfigureAwait(false); } /// diff --git a/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Loading.cs b/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Loading.cs index 2d24b6a9..06937c3e 100644 --- a/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Loading.cs +++ b/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Loading.cs @@ -1,5 +1,4 @@ using System.Buffers; -using System.IO.Compression; using System.Text; using ExcelReader.Core.Reader.Internal; using ExcelReader.Core.Reader.Zip; @@ -76,15 +75,21 @@ private void EnsureSharedLoaded() return; } _sharedLoaded = true; - ZipArchiveEntry? entry = _zip!.GetEntry("xl/sharedStrings.xml"); - if (entry is null) + if (!_zip.TryGetEntry("xl/sharedStrings.xml"u8, out ZipEntryRef entry)) { return; } - WorkbookLookups.ThrowIfSharedEntryTooLarge(entry.Length, _decompressedBytes, _options); - using LimitedReadStream stream = WorkbookLookups.OpenEntryStream(entry, _decompressedBytes, _options, + WorkbookLookups.ThrowIfSharedEntryTooLarge(entry.UncompressedSize, _decompressedBytes, _options); + if (_zip.HasMemory) + { + using ZipPart part = _zip.OpenPart(entry, _decompressedBytes, + nameof(ExcelReaderOptions.MaxSharedStringBytes), _options.MaxSharedStringBytes); + ParseSharedFromMemory(part.Memory, entry.UncompressedSize); + return; + } + using LimitedReadStream stream = _zip.OpenEntryStream(entry, _decompressedBytes, _options, nameof(ExcelReaderOptions.MaxSharedStringBytes), _options.MaxSharedStringBytes); - ParseSharedStreaming(stream, entry.Length); + ParseSharedStreaming(stream, entry.UncompressedSize); } private async ValueTask EnsureSharedLoadedAsync(CancellationToken ct) @@ -93,19 +98,23 @@ private async ValueTask EnsureSharedLoadedAsync(CancellationToken ct) { return; } + if (_zip.HasMemory) + { + EnsureSharedLoaded(); + return; + } _sharedLoaded = true; - ZipArchiveEntry? entry = _zip!.GetEntry("xl/sharedStrings.xml"); - if (entry is null) + if (!_zip.TryGetEntry("xl/sharedStrings.xml"u8, out ZipEntryRef entry)) { return; } - WorkbookLookups.ThrowIfSharedEntryTooLarge(entry.Length, _decompressedBytes, _options); - LimitedReadStream stream = await WorkbookLookups.OpenEntryStreamAsync( + WorkbookLookups.ThrowIfSharedEntryTooLarge(entry.UncompressedSize, _decompressedBytes, _options); + LimitedReadStream stream = await _zip.OpenEntryStreamAsync( entry, _decompressedBytes, _options, ct, nameof(ExcelReaderOptions.MaxSharedStringBytes), _options.MaxSharedStringBytes).ConfigureAwait(false); await using (stream.ConfigureAwait(false)) { - await ParseSharedStreamingAsync(stream, entry.Length, ct).ConfigureAwait(false); + await ParseSharedStreamingAsync(stream, entry.UncompressedSize, ct).ConfigureAwait(false); } } diff --git a/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Memory.cs b/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Memory.cs index 608f9c11..64672c4a 100644 --- a/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Memory.cs +++ b/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Memory.cs @@ -6,50 +6,6 @@ namespace ExcelReader.Core.Reader.Xlsx { public sealed partial class XlsxReader { - internal static XlsxReader CreateFromMemory(ReadOnlyMemory data, ExcelReaderOptions? options = null) - { - ExcelReaderOptions effectiveOptions = options ?? ExcelReaderOptions.Default; - return CreateFromMemory(ZipIndex.Create(data, effectiveOptions), effectiveOptions); - } - - internal static XlsxReader CreateFromMemory(ZipIndex memZip, ExcelReaderOptions effectiveOptions) - { - return ZipReaderOpen.FromMemory(memZip, zip => BuildFromMemory(zip, effectiveOptions)); - } - - private static XlsxReader BuildFromMemory(ZipIndex memZip, ExcelReaderOptions effectiveOptions) - { - DecompressedByteCounter decompressedBytes = new(effectiveOptions.MaxTotalDecompressedBytes); - using ZipPart wbPart = memZip.OpenPartOrDefault("xl/workbook.xml"u8, decompressedBytes); - using ZipPart relsPart = memZip.OpenPartOrDefault("xl/_rels/workbook.xml.rels"u8, decompressedBytes); - (string Name, string Path, ExcelSheetVisibility Visibility)[] sheets = ParseSheets(wbPart.Memory.Span, relsPart.Memory.Span); - if (sheets.Length == 0) - { - throw new InvalidDataException("The workbook contains no sheets."); - } - using ZipPart stylesPart = memZip.OpenPartOrDefault("xl/styles.xml"u8, decompressedBytes); - bool[] styleIsDate = ParseStyleDateFlags(stylesPart.Memory.Span); - bool date1904 = ParseDate1904(wbPart.Memory.Span); - return new XlsxReader(memZip, sheets, styleIsDate, date1904, effectiveOptions, decompressedBytes); - } - - private void EnsureSharedLoadedFromMemory() - { - if (_sharedLoaded) - { - return; - } - _sharedLoaded = true; - if (!_memZip!.TryGetEntry("xl/sharedStrings.xml"u8, out ZipEntryRef entry)) - { - return; - } - WorkbookLookups.ThrowIfSharedEntryTooLarge(entry.UncompressedSize, _decompressedBytes, _options); - using ZipPart part = _memZip.OpenPart(entry, _decompressedBytes, - nameof(ExcelReaderOptions.MaxSharedStringBytes), _options.MaxSharedStringBytes); - ParseSharedFromMemory(part.Memory, entry.UncompressedSize); - } - private void ParseSharedFromMemory(ReadOnlyMemory content, long entryLength) { LimitChecks.ThrowIfEntryLengthExceeds(entryLength, Array.MaxLength, "ArrayMaxLength"); @@ -59,11 +15,5 @@ private void ParseSharedFromMemory(ReadOnlyMemory content, long entryLengt _sharedFlat = ArrayPool.Shared.Rent(Math.Max(1, partLength)); _sharedOffsets = ParseSharedBody(io, stream: null, partLength); } - - private Enumerator GetEnumeratorFromMemory() - { - ZipEntryRef entry = WorkbookLookups.GetWorksheetEntry(_memZip!, _sheets[_current].Path); - return new Enumerator(this, _memZip!.OpenEntryStream(entry, _decompressedBytes, _options), entry.UncompressedSize); - } } } diff --git a/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.cs b/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.cs index 220c6cee..316601a4 100644 --- a/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.cs +++ b/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.cs @@ -1,6 +1,6 @@ using System.Buffers; -using System.IO.Compression; using ExcelReader.Core.Reader.Internal; +using ExcelReader.Core.Reader.Sources; using ExcelReader.Core.Reader.Zip; namespace ExcelReader.Core.Reader.Xlsx @@ -8,10 +8,7 @@ namespace ExcelReader.Core.Reader.Xlsx /// Reads rows from an Office Open XML (.xlsx) workbook, streaming each sheet's cells without loading the whole file into memory. public sealed partial class XlsxReader : IExcelRowReader, IExcelRowReader { - private readonly Stream? _stream; - private readonly bool _leaveOpen; - private readonly ZipArchive? _zip; - private readonly ZipIndex? _memZip; + private readonly ZipIndex _zip; private readonly ExcelReaderOptions _options; private readonly DecompressedByteCounter _decompressedBytes; private readonly (string Name, string Path, ExcelSheetVisibility Visibility)[] _sheets; @@ -24,48 +21,39 @@ public sealed partial class XlsxReader : IExcelRowReader, IExcelRowReader data, ExcelReaderOptions? options = null) { - _leaveOpen = true; - _memZip = memZip; - _options = options; - _decompressedBytes = decompressedBytes; - _sheets = sheets; - _styleIsDate = styleIsDate; - IsDate1904 = date1904; + ExcelReaderOptions effectiveOptions = options ?? ExcelReaderOptions.Default; + return new XlsxReader(ZipIndex.Create(data, effectiveOptions), effectiveOptions); } - internal static ValueTask CreateAsync(Stream stream, bool leaveOpen, ExcelReaderOptions? options = null, CancellationToken ct = default) + internal static XlsxReader CreateFromIndex(ZipIndex zip, ExcelReaderOptions options) { - ExcelReaderOptions effectiveOptions = options ?? ExcelReaderOptions.Default; - DecompressedByteCounter decompressedBytes = new(effectiveOptions.MaxTotalDecompressedBytes); - return ZipReaderOpen.OpenAsync(stream, leaveOpen, effectiveOptions, - zip => ParseAsync(stream, leaveOpen, zip, effectiveOptions, decompressedBytes, ct), ct); + return new XlsxReader(zip, options); } - internal static ValueTask CreateFromOpenZipAsync( - Stream stream, bool leaveOpen, ZipArchive zip, ExcelReaderOptions? options, CancellationToken ct) + internal static async ValueTask CreateAsync(Stream stream, bool leaveOpen, ExcelReaderOptions? options = null, CancellationToken ct = default) { ExcelReaderOptions effectiveOptions = options ?? ExcelReaderOptions.Default; - DecompressedByteCounter decompressedBytes = new(effectiveOptions.MaxTotalDecompressedBytes); - return ZipReaderOpen.FromOpenZipAsync(stream, leaveOpen, zip, effectiveOptions, - z => ParseAsync(stream, leaveOpen, z, effectiveOptions, decompressedBytes, ct)); + ByteSource source = await ByteSource.FromStreamAsync(stream, leaveOpen, ct).ConfigureAwait(false); + ZipIndex zip = await ZipIndex.CreateAsync(source, effectiveOptions, ct).ConfigureAwait(false); + return await CreateFromIndexAsync(zip, effectiveOptions, ct).ConfigureAwait(false); } - private static async ValueTask ParseAsync( - Stream stream, bool leaveOpen, ZipArchive zip, ExcelReaderOptions effectiveOptions, - DecompressedByteCounter decompressedBytes, CancellationToken ct) + internal static async ValueTask CreateFromIndexAsync(ZipIndex zip, ExcelReaderOptions options, CancellationToken ct) { - using ZipPart wbPart = await ZipEntryBytes.ReadAsync(zip, "xl/workbook.xml", decompressedBytes, ct).ConfigureAwait(false); - using ZipPart relsPart = await ZipEntryBytes.ReadAsync(zip, "xl/_rels/workbook.xml.rels", decompressedBytes, ct).ConfigureAwait(false); - var sheets = ParseSheets(wbPart.Memory.Span, relsPart.Memory.Span); - if (sheets.Length == 0) + try { - throw new InvalidDataException("The workbook contains no sheets."); + DecompressedByteCounter decompressedBytes = new(options.MaxTotalDecompressedBytes); + using ZipPart wbPart = await zip.OpenPartOrDefaultAsync("xl/workbook.xml"u8, decompressedBytes, ct).ConfigureAwait(false); + using ZipPart relsPart = await zip.OpenPartOrDefaultAsync("xl/_rels/workbook.xml.rels"u8, decompressedBytes, ct).ConfigureAwait(false); + (string Name, string Path, ExcelSheetVisibility Visibility)[] sheets = ParseSheets(wbPart.Memory.Span, relsPart.Memory.Span); + if (sheets.Length == 0) + { + throw new InvalidDataException("The workbook contains no sheets."); + } + using ZipPart stylesPart = await zip.OpenPartOrDefaultAsync("xl/styles.xml"u8, decompressedBytes, ct).ConfigureAwait(false); + bool[] styleIsDate = ParseStyleDateFlags(stylesPart.Memory.Span); + bool date1904 = ParseDate1904(wbPart.Memory.Span); + return new XlsxReader(zip, sheets, styleIsDate, date1904, options, decompressedBytes); + } + catch + { + zip.Dispose(); + throw; } - using ZipPart stylesPart = await ZipEntryBytes.ReadAsync(zip, "xl/styles.xml", decompressedBytes, ct).ConfigureAwait(false); - var styleIsDate = ParseStyleDateFlags(stylesPart.Memory.Span); - bool date1904 = ParseDate1904(wbPart.Memory.Span); - return new XlsxReader(stream, leaveOpen, zip, sheets, styleIsDate, date1904, effectiveOptions, decompressedBytes); } /// @@ -168,14 +152,9 @@ public void MoveToSheet(int index) /// public Enumerator GetEnumerator() { - if (_memZip is not null) - { - EnsureSharedLoadedFromMemory(); - return GetEnumeratorFromMemory(); - } EnsureSharedLoaded(); - var entry = WorkbookLookups.GetWorksheetEntry(_zip!, _sheets[_current].Path); - return new Enumerator(this, WorkbookLookups.OpenEntryStream(entry, _decompressedBytes, _options), entry.Length); + ZipEntryRef entry = WorkbookLookups.GetWorksheetEntry(_zip, _sheets[_current].Path); + return new Enumerator(this, _zip.OpenEntryStream(entry, _decompressedBytes, _options), entry.UncompressedSize); } IExcelRowEnumerator IExcelRowReader.GetEnumerator() @@ -186,12 +165,11 @@ IExcelRowEnumerator IExcelRowReader.GetEnumerator() /// public Enumerator GetAsyncEnumerator(CancellationToken ct = default) { - if (_memZip is not null) + if (_zip.HasMemory) { - EnsureSharedLoadedFromMemory(); - return GetEnumeratorFromMemory(); + return GetEnumerator(); } - return new Enumerator(this, WorkbookLookups.GetWorksheetEntry(_zip!, _sheets[_current].Path), ct); + return new Enumerator(this, WorkbookLookups.GetWorksheetEntry(_zip, _sheets[_current].Path), ct); } IExcelRowEnumerator IExcelRowReader.GetAsyncEnumerator(CancellationToken ct) @@ -211,31 +189,14 @@ public void Dispose() ArrayPool.Shared.Return(_sharedFlat); _sharedFlat = []; } - _memZip?.Dispose(); - _zip?.Dispose(); - if (!_leaveOpen) - { - _stream?.Dispose(); - } + _zip.Dispose(); } /// - public async ValueTask DisposeAsync() + public ValueTask DisposeAsync() { - if (_sharedFlat.Length > 0) - { - ArrayPool.Shared.Return(_sharedFlat); - _sharedFlat = []; - } - _memZip?.Dispose(); - if (_zip is not null) - { - await _zip.DisposeAsync().ConfigureAwait(false); - } - if (!_leaveOpen && _stream is not null) - { - await _stream.DisposeAsync().ConfigureAwait(false); - } + Dispose(); + return ValueTask.CompletedTask; } } } diff --git a/tests/ExcelReader.Tests/Reader/SourceParityTests.cs b/tests/ExcelReader.Tests/Reader/SourceParityTests.cs new file mode 100644 index 00000000..d05a6c5f --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/SourceParityTests.cs @@ -0,0 +1,205 @@ +using System.Globalization; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Xlsx; +using ExcelReader.Core.Writer.Xlsx; + +namespace ExcelReader.Tests.Reader +{ + public sealed class SourceParityTests : IDisposable + { + public enum Kind + { + Memory, + ExposedMemoryStream, + OpaqueMemoryStream, + Trickle, + FileStream, + FilePath, + NonSeekable, + } + + private readonly List _paths = []; + + public void Dispose() + { + foreach (string path in _paths) + { + File.Delete(path); + } + } + + public static TheoryData Kinds => + [ + Kind.Memory, Kind.ExposedMemoryStream, Kind.OpaqueMemoryStream, Kind.Trickle, + Kind.FileStream, Kind.FilePath, Kind.NonSeekable, + ]; + + private static byte[] BuildXlsx() + { + using MemoryStream buffer = new(); + using (XlsxWorkbookWriter workbook = XlsxWorkbookWriter.Create(buffer, leaveOpen: true, new XlsxWriterOptions { UseSharedStrings = true })) + { + for (int s = 0; s < 3; s++) + { + XlsxSheetWriter sheet = workbook.AddSheet("sheet" + s); + for (int r = 0; r < 400; r++) + { + using XlsxRowWriter row = sheet.StartRow(); + row.Write("name-" + ((s * 1000) + (r % 50)).ToString(CultureInfo.InvariantCulture)); + row.Write((s * 100000) + r); + row.Write(r % 2 == 0); + } + sheet.End(); + sheet.Dispose(); + } + workbook.End(); + } + return buffer.ToArray(); + } + + private string WriteTemp(byte[] bytes, string extension) + { + string path = Path.Combine(Path.GetTempPath(), "excelreader-parity-" + Guid.NewGuid().ToString("N") + extension); + File.WriteAllBytes(path, bytes); + _paths.Add(path); + return path; + } + + private XlsxReader OpenXlsx(Kind kind, byte[] bytes) + { + switch (kind) + { + case Kind.Memory: + return Excel.FromXlsx(bytes); + case Kind.ExposedMemoryStream: + MemoryStream exposed = new(); + exposed.Write(bytes); + exposed.Position = 0; + return Excel.FromXlsx(exposed, leaveOpen: false); + case Kind.OpaqueMemoryStream: + return Excel.FromXlsx(new MemoryStream(bytes, writable: false), leaveOpen: false); + case Kind.Trickle: + return Excel.FromXlsx(new TrickleStream(bytes), leaveOpen: false); + case Kind.FileStream: + return Excel.FromXlsx(File.OpenRead(WriteTemp(bytes, ".xlsx")), leaveOpen: false); + case Kind.FilePath: + return Excel.FromXlsxFile(WriteTemp(bytes, ".xlsx")); + default: + return Excel.FromXlsx(new NonSeekableStream(bytes), leaveOpen: false); + } + } + + private static List Dump(IExcelRowReader reader) + { + List lines = []; + for (int s = 0; s < reader.SheetCount; s++) + { + reader.MoveToSheet(s); + using IExcelRowEnumerator e = reader.GetEnumerator(); + while (e.MoveNext()) + { + lines.Add(Line(s, e.Current)); + } + } + return lines; + } + + private static async Task> DumpAsync(IExcelRowReader reader) + { + List lines = []; + for (int s = 0; s < reader.SheetCount; s++) + { + reader.MoveToSheet(s); + await using IExcelRowEnumerator e = reader.GetAsyncEnumerator(TestContext.Current.CancellationToken); + while (await e.MoveNextAsync()) + { + lines.Add(Line(s, e.Current)); + } + } + return lines; + } + + private static string Line(int sheet, Row row) + { + string[] cells = new string[row.ColumnCount]; + for (int c = 0; c < cells.Length; c++) + { + cells[c] = row[c].GetString(); + } + return sheet.ToString(CultureInfo.InvariantCulture) + "|" + string.Join("|", cells); + } + + [Theory] + [MemberData(nameof(Kinds))] + public void Xlsx_Reads_The_Same_Rows_From_Every_Source(Kind kind) + { + byte[] bytes = BuildXlsx(); + using IExcelRowReader expected = Excel.FromXlsx(bytes); + using IExcelRowReader actual = OpenXlsx(kind, bytes); + Assert.Equal(Dump(expected), Dump(actual)); + } + + [Theory] + [MemberData(nameof(Kinds))] + public async Task Xlsx_Reads_The_Same_Rows_Asynchronously_From_Every_Source(Kind kind) + { + byte[] bytes = BuildXlsx(); + using IExcelRowReader expected = Excel.FromXlsx(bytes); + using IExcelRowReader actual = OpenXlsx(kind, bytes); + Assert.Equal(Dump(expected), await DumpAsync(actual)); + } + + [Theory] + [MemberData(nameof(Kinds))] + public void Xlsx_Interleaved_Enumerators_Keep_Their_Own_Position(Kind kind) + { + byte[] bytes = BuildXlsx(); + using IExcelRowReader reader = OpenXlsx(kind, bytes); + reader.MoveToSheet(0); + using IExcelRowEnumerator first = reader.GetEnumerator(); + reader.MoveToSheet(1); + using IExcelRowEnumerator second = reader.GetEnumerator(); + for (int r = 0; r < 400; r++) + { + Assert.True(first.MoveNext()); + Assert.True(second.MoveNext()); + Assert.Equal("name-" + (r % 50).ToString(CultureInfo.InvariantCulture), first.Current[0].GetString()); + Assert.Equal("name-" + (1000 + (r % 50)).ToString(CultureInfo.InvariantCulture), second.Current[0].GetString()); + } + Assert.False(first.MoveNext()); + Assert.False(second.MoveNext()); + } + + [Fact] + public void Xlsx_File_Can_Be_Deleted_After_Dispose() + { + string path = WriteTemp(BuildXlsx(), ".xlsx"); + using (IExcelRowReader reader = Excel.FromXlsxFile(path)) + { + Assert.NotEmpty(Dump(reader)); + } + File.Delete(path); + Assert.False(File.Exists(path)); + } + + [Fact] + public void Xlsx_Leaves_A_Borrowed_Stream_Usable() + { + byte[] bytes = BuildXlsx(); + using TrickleStream stream = new(bytes); + using (IExcelRowReader reader = Excel.FromXlsx(stream, leaveOpen: true)) + { + Assert.NotEmpty(Dump(reader)); + } + Assert.True(stream.CanRead); + } + + [Fact] + public void Xlsx_Cut_Short_Is_Invalid_Data() + { + byte[] bytes = BuildXlsx(); + string path = WriteTemp(bytes[..(bytes.Length / 2)], ".xlsx"); + Assert.Throws(() => Excel.FromXlsxFile(path)); + } + } +} From 676d1465fa54099b899c779f6877157eb83154e1 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Tue, 6 Oct 2026 01:32:04 -0300 Subject: [PATCH 08/28] refactor(core): read XLSB from streams and files through ZipIndex Co-Authored-By: Claude Sonnet 5.5 --- src/ExcelReader.Core/Reader/Excel.cs | 25 ++- .../Reader/Xlsb/XlsbReader.Enumerator.cs | 12 +- .../Reader/Xlsb/XlsbReader.Memory.cs | 55 ------ .../Reader/Xlsb/XlsbReader.cs | 186 +++++++----------- .../Reader/SourceParityTests.cs | 89 +++++++++ 5 files changed, 189 insertions(+), 178 deletions(-) delete mode 100644 src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Memory.cs diff --git a/src/ExcelReader.Core/Reader/Excel.cs b/src/ExcelReader.Core/Reader/Excel.cs index a82116e7..59b767f8 100644 --- a/src/ExcelReader.Core/Reader/Excel.cs +++ b/src/ExcelReader.Core/Reader/Excel.cs @@ -376,7 +376,7 @@ public static IExcelRowReader Open(ReadOnlyMemory data, ExcelReaderOptions return format switch { ExcelFileFormat.Xls => new XlsReader(data, effective), - ExcelFileFormat.Xlsb => XlsbReader.CreateFromMemory(memZip!, effective), + ExcelFileFormat.Xlsb => XlsbReader.CreateFromIndex(memZip!, effective), ExcelFileFormat.Xlsx => XlsxReader.CreateFromIndex(memZip!, effective), _ => throw new System.Diagnostics.UnreachableException(), }; @@ -392,7 +392,7 @@ private static IExcelRowReader OpenFromPlainMemory(ReadOnlyMemory plain, E } return format switch { - ExcelFileFormat.Xlsb => XlsbReader.CreateFromMemory(memZip!, options), + ExcelFileFormat.Xlsb => XlsbReader.CreateFromIndex(memZip!, options), ExcelFileFormat.Xlsx => XlsxReader.CreateFromIndex(memZip!, options), _ => throw new System.Diagnostics.UnreachableException(), }; @@ -689,7 +689,7 @@ private static IExcelRowReader OpenSeekable(Stream stream, bool leaveOpen, Excel return format switch { ExcelFileFormat.Xls => new XlsReader(stream, leaveOpen, options), - ExcelFileFormat.Xlsb => new XlsbReader(stream, leaveOpen, zip!, options), + ExcelFileFormat.Xlsb => ReopenXlsb(stream, leaveOpen, zip!, options), ExcelFileFormat.Xlsx => ReopenXlsx(stream, leaveOpen, zip!, options), _ => throw new System.Diagnostics.UnreachableException(), }; @@ -704,7 +704,7 @@ private static IExcelRowReader OpenDecryptedZip(Stream decrypted, ExcelReaderOpt zip = zipPeek; return zipFormat switch { - ExcelFileFormat.Xlsb => new XlsbReader(decrypted, leaveOpen: false, zip, options), + ExcelFileFormat.Xlsb => ReopenXlsb(decrypted, leaveOpen: false, zip, options), ExcelFileFormat.Xlsx => ReopenXlsx(decrypted, leaveOpen: false, zip, options), _ => throw new System.Diagnostics.UnreachableException(), }; @@ -748,7 +748,7 @@ private static async ValueTask OpenSeekableAsync(Stream stream, return format switch { ExcelFileFormat.Xls => await XlsReader.CreateAsync(stream, leaveOpen, options, ct).ConfigureAwait(false), - ExcelFileFormat.Xlsb => await XlsbReader.CreateFromOpenZipAsync(stream, leaveOpen, zip!, options, ct).ConfigureAwait(false), + ExcelFileFormat.Xlsb => await ReopenXlsbAsync(stream, leaveOpen, zip!, options, ct).ConfigureAwait(false), ExcelFileFormat.Xlsx => await ReopenXlsxAsync(stream, leaveOpen, zip!, options, ct).ConfigureAwait(false), _ => throw new System.Diagnostics.UnreachableException(), }; @@ -763,7 +763,7 @@ private static async ValueTask OpenDecryptedZipAsync(Stream dec zip = zipPeek; return zipFormat switch { - ExcelFileFormat.Xlsb => await XlsbReader.CreateFromOpenZipAsync(decrypted, leaveOpen: false, zip, options, ct).ConfigureAwait(false), + ExcelFileFormat.Xlsb => await ReopenXlsbAsync(decrypted, leaveOpen: false, zip, options, ct).ConfigureAwait(false), ExcelFileFormat.Xlsx => await ReopenXlsxAsync(decrypted, leaveOpen: false, zip, options, ct).ConfigureAwait(false), _ => throw new System.Diagnostics.UnreachableException(), }; @@ -829,6 +829,19 @@ private static async ValueTask ReopenXlsxAsync( return await XlsxReader.CreateAsync(stream, leaveOpen, options, ct).ConfigureAwait(false); } + private static XlsbReader ReopenXlsb(Stream stream, bool leaveOpen, ZipArchive peeked, ExcelReaderOptions? options) + { + peeked.Dispose(); + return new XlsbReader(stream, leaveOpen, options); + } + + private static async ValueTask ReopenXlsbAsync( + Stream stream, bool leaveOpen, ZipArchive peeked, ExcelReaderOptions? options, CancellationToken ct) + { + await peeked.DisposeAsync().ConfigureAwait(false); + return await XlsbReader.CreateAsync(stream, leaveOpen, options, ct).ConfigureAwait(false); + } + private static ExcelFileFormat ClassifyZipStream(Stream stream, long start, out ZipArchive zip) { var zipPeek = new ZipArchive(stream, ZipArchiveMode.Read, leaveOpen: true); diff --git a/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Enumerator.cs b/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Enumerator.cs index 5cc77df6..69deabae 100644 --- a/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Enumerator.cs +++ b/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Enumerator.cs @@ -1,8 +1,8 @@ using System.Diagnostics.CodeAnalysis; -using System.IO.Compression; using System.Runtime.CompilerServices; using System.Text; using ExcelReader.Core.Reader.Internal; +using ExcelReader.Core.Reader.Zip; namespace ExcelReader.Core.Reader.Xlsb { @@ -24,7 +24,7 @@ public sealed class Enumerator : PooledStreamRowEnumerator, IExcelRowEnumerator private readonly bool[] _styleIsDate; private readonly int[] _sharedOffsets; private readonly Utf8StringCache? _contentCache; - private readonly ZipArchiveEntry? _entry; + private readonly ZipEntryRef _entry; private bool _ended; private bool _pendingRowHdr; @@ -37,8 +37,8 @@ internal Enumerator(XlsbReader reader, Stream sheet, long entryLength = 0, Cance _contentCache = reader._options.InternStrings ? new Utf8StringCache() : null; } - internal Enumerator(XlsbReader reader, ZipArchiveEntry entry, CancellationToken ct) - : base(reader._options.MaxCellBytes, nameof(ExcelReaderOptions.MaxCellBytes), WorkbookLookups.InitialBufferCapacity(entry.Length), ct) + internal Enumerator(XlsbReader reader, ZipEntryRef entry, CancellationToken ct) + : base(reader._options.MaxCellBytes, nameof(ExcelReaderOptions.MaxCellBytes), WorkbookLookups.InitialBufferCapacity(entry.UncompressedSize), ct) { _reader = reader; _styleIsDate = reader._styleIsDate; @@ -49,12 +49,12 @@ internal Enumerator(XlsbReader reader, ZipArchiveEntry entry, CancellationToken private protected override Stream OpenSource() { - return WorkbookLookups.OpenEntryStream(_entry!, _reader._decompressedBytes, _reader._options); + return _reader._zip!.OpenEntryStream(_entry, _reader._decompressedBytes, _reader._options); } private protected override async ValueTask OpenSourceAsync() { - return await WorkbookLookups.OpenEntryStreamAsync(_entry!, _reader._decompressedBytes, _reader._options, _ct).ConfigureAwait(false); + return await _reader._zip!.OpenEntryStreamAsync(_entry, _reader._decompressedBytes, _reader._options, _ct).ConfigureAwait(false); } /// diff --git a/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Memory.cs b/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Memory.cs deleted file mode 100644 index 14da0cee..00000000 --- a/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Memory.cs +++ /dev/null @@ -1,55 +0,0 @@ -using ExcelReader.Core.Reader.Internal; -using ExcelReader.Core.Reader.Zip; - -namespace ExcelReader.Core.Reader.Xlsb -{ - public sealed partial class XlsbReader - { - internal static XlsbReader CreateFromMemory(ReadOnlyMemory data, ExcelReaderOptions? options = null) - { - ExcelReaderOptions effectiveOptions = options ?? ExcelReaderOptions.Default; - return CreateFromMemory(ZipIndex.Create(data, effectiveOptions), effectiveOptions); - } - - internal static XlsbReader CreateFromMemory(ZipIndex memZip, ExcelReaderOptions effectiveOptions) - { - return ZipReaderOpen.FromMemory(memZip, zip => BuildFromMemory(zip, effectiveOptions)); - } - - private static XlsbReader BuildFromMemory(ZipIndex memZip, ExcelReaderOptions effectiveOptions) - { - DecompressedByteCounter decompressedBytes = new(effectiveOptions.MaxTotalDecompressedBytes); - using ZipPart wbPart = memZip.OpenPartOrDefault("xl/workbook.bin"u8, decompressedBytes); - using ZipPart relsPart = memZip.OpenPartOrDefault("xl/_rels/workbook.bin.rels"u8, decompressedBytes); - (string Name, string Path, ExcelSheetVisibility Visibility)[] sheets = XlsbWorkbook.ParseSheets(wbPart.Memory.Span, relsPart.Memory.Span); - if (sheets.Length == 0) - { - throw new InvalidDataException("The workbook contains no sheets."); - } - using ZipPart stylesPart = memZip.OpenPartOrDefault("xl/styles.bin"u8, decompressedBytes); - bool[] styleIsDate = XlsbStyles.ParseStyleDateFlags(stylesPart.Memory.Span); - bool date1904 = XlsbWorkbook.ParseDate1904(wbPart.Memory.Span); - (byte[] flat, int[] offsets) = LoadSharedStringsFromMemory(memZip, decompressedBytes, effectiveOptions); - return new XlsbReader(memZip, sheets, styleIsDate, date1904, flat, offsets, effectiveOptions, decompressedBytes); - } - - private static (byte[] Flat, int[] Offsets) LoadSharedStringsFromMemory( - ZipIndex memZip, DecompressedByteCounter decompressedBytes, ExcelReaderOptions options) - { - if (!memZip.TryGetEntry("xl/sharedStrings.bin"u8, out ZipEntryRef entry)) - { - return ([], [0]); - } - WorkbookLookups.ThrowIfSharedEntryTooLarge(entry.UncompressedSize, decompressedBytes, options); - using ZipPart part = memZip.OpenPart(entry, decompressedBytes, - nameof(ExcelReaderOptions.MaxSharedStringBytes), options.MaxSharedStringBytes); - return XlsbSharedStrings.Parse(part.Memory.Span, options); - } - - private Enumerator GetEnumeratorFromMemory() - { - ZipEntryRef entry = WorkbookLookups.GetWorksheetEntry(_memZip!, _sheets![_current].Path); - return new Enumerator(this, _memZip!.OpenEntryStream(entry, _decompressedBytes, _options), entry.UncompressedSize); - } - } -} diff --git a/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.cs b/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.cs index 0c81dfdc..a07bd97e 100644 --- a/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.cs +++ b/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.cs @@ -1,6 +1,6 @@ using System.Buffers; -using System.IO.Compression; using ExcelReader.Core.Reader.Internal; +using ExcelReader.Core.Reader.Sources; using ExcelReader.Core.Reader.Zip; namespace ExcelReader.Core.Reader.Xlsb @@ -21,10 +21,7 @@ public sealed partial class XlsbReader : IExcelRowReader, IExcelRowReader data, ExcelReaderOptions? options = null) { - _leaveOpen = true; - _memZip = memZip; - _options = options; - _decompressedBytes = decompressedBytes; - _sheets = sheets; - _styleIsDate = styleIsDate; - IsDate1904 = date1904; - _sharedFlat = sharedFlat; - _sharedOffsets = sharedOffsets; - _pooledSharedFlat = false; + ExcelReaderOptions effectiveOptions = options ?? ExcelReaderOptions.Default; + return new XlsbReader(ZipIndex.Create(data, effectiveOptions), effectiveOptions); } - internal static ValueTask CreateAsync(Stream stream, bool leaveOpen, ExcelReaderOptions? options = null, CancellationToken ct = default) + internal static XlsbReader CreateFromIndex(ZipIndex zip, ExcelReaderOptions options) { - ExcelReaderOptions effectiveOptions = options ?? ExcelReaderOptions.Default; - DecompressedByteCounter decompressedBytes = new(effectiveOptions.MaxTotalDecompressedBytes); - return ZipReaderOpen.OpenAsync(stream, leaveOpen, effectiveOptions, - zip => ParseAsync(stream, leaveOpen, zip, effectiveOptions, decompressedBytes, ct), ct); + return new XlsbReader(zip, options); } - internal static ValueTask CreateFromOpenZipAsync( - Stream stream, bool leaveOpen, ZipArchive zip, ExcelReaderOptions? options, CancellationToken ct) + internal static async ValueTask CreateAsync(Stream stream, bool leaveOpen, ExcelReaderOptions? options = null, CancellationToken ct = default) { ExcelReaderOptions effectiveOptions = options ?? ExcelReaderOptions.Default; - DecompressedByteCounter decompressedBytes = new(effectiveOptions.MaxTotalDecompressedBytes); - return ZipReaderOpen.FromOpenZipAsync(stream, leaveOpen, zip, effectiveOptions, - z => ParseAsync(stream, leaveOpen, z, effectiveOptions, decompressedBytes, ct)); + ByteSource source = await ByteSource.FromStreamAsync(stream, leaveOpen, ct).ConfigureAwait(false); + ZipIndex zip = await ZipIndex.CreateAsync(source, effectiveOptions, ct).ConfigureAwait(false); + return await CreateFromIndexAsync(zip, effectiveOptions, ct).ConfigureAwait(false); } - private static async ValueTask ParseAsync( - Stream stream, bool leaveOpen, ZipArchive zip, ExcelReaderOptions effectiveOptions, - DecompressedByteCounter decompressedBytes, CancellationToken ct) + internal static async ValueTask CreateFromIndexAsync(ZipIndex zip, ExcelReaderOptions options, CancellationToken ct) { - using ZipPart wbPart = await ZipEntryBytes.ReadAsync(zip, "xl/workbook.bin", decompressedBytes, ct).ConfigureAwait(false); - using ZipPart relsPart = await ZipEntryBytes.ReadAsync(zip, "xl/_rels/workbook.bin.rels", decompressedBytes, ct).ConfigureAwait(false); - var sheets = XlsbWorkbook.ParseSheets(wbPart.Memory.Span, relsPart.Memory.Span); - if (sheets.Length == 0) + try + { + DecompressedByteCounter decompressedBytes = new(options.MaxTotalDecompressedBytes); + using ZipPart wbPart = await zip.OpenPartOrDefaultAsync("xl/workbook.bin"u8, decompressedBytes, ct).ConfigureAwait(false); + using ZipPart relsPart = await zip.OpenPartOrDefaultAsync("xl/_rels/workbook.bin.rels"u8, decompressedBytes, ct).ConfigureAwait(false); + (string Name, string Path, ExcelSheetVisibility Visibility)[] sheets = XlsbWorkbook.ParseSheets(wbPart.Memory.Span, relsPart.Memory.Span); + if (sheets.Length == 0) + { + throw new InvalidDataException("The workbook contains no sheets."); + } + using ZipPart stylesPart = await zip.OpenPartOrDefaultAsync("xl/styles.bin"u8, decompressedBytes, ct).ConfigureAwait(false); + bool[] styleIsDate = XlsbStyles.ParseStyleDateFlags(stylesPart.Memory.Span); + bool date1904 = XlsbWorkbook.ParseDate1904(wbPart.Memory.Span); + (byte[] flat, int[] offsets, bool pooled) = await LoadSharedStringsAsync(zip, decompressedBytes, options, ct).ConfigureAwait(false); + return new XlsbReader(zip, sheets, styleIsDate, date1904, flat, offsets, pooled, options, decompressedBytes); + } + catch { - throw new InvalidDataException("The workbook contains no sheets."); + zip.Dispose(); + throw; } - using ZipPart stylesPart = await ZipEntryBytes.ReadAsync(zip, "xl/styles.bin", decompressedBytes, ct).ConfigureAwait(false); - var styleIsDate = XlsbStyles.ParseStyleDateFlags(stylesPart.Memory.Span); - bool date1904 = XlsbWorkbook.ParseDate1904(wbPart.Memory.Span); - var (flat, offsets) = await LoadSharedStringsAsync(zip, decompressedBytes, effectiveOptions, ct).ConfigureAwait(false); - return new XlsbReader(stream, leaveOpen, zip, sheets, styleIsDate, date1904, flat, offsets, effectiveOptions, decompressedBytes); } - private (byte[] Flat, int[] Offsets) LoadSharedStrings(ZipArchive zip) + private static (byte[] Flat, int[] Offsets, bool Pooled) LoadSharedStrings( + ZipIndex zip, DecompressedByteCounter decompressedBytes, ExcelReaderOptions options) { - ZipArchiveEntry? entry = zip.GetEntry("xl/sharedStrings.bin"); - if (entry is null) + if (!zip.TryGetEntry("xl/sharedStrings.bin"u8, out ZipEntryRef entry)) { - return ([], [0]); + return ([], [0], false); } - WorkbookLookups.ThrowIfSharedEntryTooLarge(entry.Length, _decompressedBytes, _options); - using LimitedReadStream stream = WorkbookLookups.OpenEntryStream(entry, _decompressedBytes, _options, - nameof(ExcelReaderOptions.MaxSharedStringBytes), _options.MaxSharedStringBytes); - return XlsbSharedStrings.ParseStreaming(stream, entry.Length, _options); + WorkbookLookups.ThrowIfSharedEntryTooLarge(entry.UncompressedSize, decompressedBytes, options); + if (zip.HasMemory) + { + using ZipPart part = zip.OpenPart(entry, decompressedBytes, + nameof(ExcelReaderOptions.MaxSharedStringBytes), options.MaxSharedStringBytes); + (byte[] parsedFlat, int[] parsedOffsets) = XlsbSharedStrings.Parse(part.Memory.Span, options); + return (parsedFlat, parsedOffsets, false); + } + using LimitedReadStream stream = zip.OpenEntryStream(entry, decompressedBytes, options, + nameof(ExcelReaderOptions.MaxSharedStringBytes), options.MaxSharedStringBytes); + (byte[] flat, int[] offsets) = XlsbSharedStrings.ParseStreaming(stream, entry.UncompressedSize, options); + return (flat, offsets, flat.Length != 0); } - private static async ValueTask<(byte[] Flat, int[] Offsets)> LoadSharedStringsAsync( - ZipArchive zip, DecompressedByteCounter decompressedBytes, ExcelReaderOptions options, CancellationToken ct) + private static async ValueTask<(byte[] Flat, int[] Offsets, bool Pooled)> LoadSharedStringsAsync( + ZipIndex zip, DecompressedByteCounter decompressedBytes, ExcelReaderOptions options, CancellationToken ct) { - ZipArchiveEntry? entry = zip.GetEntry("xl/sharedStrings.bin"); - if (entry is null) + if (zip.HasMemory || !zip.TryGetEntry("xl/sharedStrings.bin"u8, out ZipEntryRef entry)) { - return ([], [0]); + return LoadSharedStrings(zip, decompressedBytes, options); } - WorkbookLookups.ThrowIfSharedEntryTooLarge(entry.Length, decompressedBytes, options); - LimitedReadStream stream = await WorkbookLookups.OpenEntryStreamAsync(entry, decompressedBytes, options, ct, + WorkbookLookups.ThrowIfSharedEntryTooLarge(entry.UncompressedSize, decompressedBytes, options); + LimitedReadStream stream = await zip.OpenEntryStreamAsync(entry, decompressedBytes, options, ct, nameof(ExcelReaderOptions.MaxSharedStringBytes), options.MaxSharedStringBytes).ConfigureAwait(false); await using (stream.ConfigureAwait(false)) { - return await XlsbSharedStrings.ParseStreamingAsync(stream, entry.Length, options, ct).ConfigureAwait(false); + (byte[] flat, int[] offsets) = await XlsbSharedStrings.ParseStreamingAsync(stream, entry.UncompressedSize, options, ct).ConfigureAwait(false); + return (flat, offsets, flat.Length != 0); } } @@ -229,12 +217,8 @@ internal bool IsDateStyle(int style) /// public Enumerator GetEnumerator() { - if (_memZip is not null) - { - return GetEnumeratorFromMemory(); - } - var entry = WorkbookLookups.GetWorksheetEntry(_zip!, _sheets![_current].Path); - return new Enumerator(this, WorkbookLookups.OpenEntryStream(entry, _decompressedBytes, _options), entry.Length); + ZipEntryRef entry = WorkbookLookups.GetWorksheetEntry(_zip!, _sheets![_current].Path); + return new Enumerator(this, _zip!.OpenEntryStream(entry, _decompressedBytes, _options), entry.UncompressedSize); } IExcelRowEnumerator IExcelRowReader.GetEnumerator() @@ -245,11 +229,11 @@ IExcelRowEnumerator IExcelRowReader.GetEnumerator() /// public Enumerator GetAsyncEnumerator(CancellationToken ct = default) { - if (_memZip is not null) + if (_zip!.HasMemory) { - return GetEnumeratorFromMemory(); + return GetEnumerator(); } - return new Enumerator(this, WorkbookLookups.GetWorksheetEntry(_zip!, _sheets![_current].Path), ct); + return new Enumerator(this, WorkbookLookups.GetWorksheetEntry(_zip, _sheets![_current].Path), ct); } IExcelRowEnumerator IExcelRowReader.GetAsyncEnumerator(CancellationToken ct) @@ -269,34 +253,14 @@ public void Dispose() { ArrayPool.Shared.Return(_sharedFlat); } - _memZip?.Dispose(); _zip?.Dispose(); - if (!_leaveOpen) - { - _stream?.Dispose(); - } } /// - public async ValueTask DisposeAsync() + public ValueTask DisposeAsync() { - if (Interlocked.Exchange(ref _disposed, 1) != 0) - { - return; - } - if (_pooledSharedFlat) - { - ArrayPool.Shared.Return(_sharedFlat); - } - _memZip?.Dispose(); - if (_zip is not null) - { - await _zip.DisposeAsync().ConfigureAwait(false); - } - if (!_leaveOpen && _stream is not null) - { - await _stream.DisposeAsync().ConfigureAwait(false); - } + Dispose(); + return ValueTask.CompletedTask; } } diff --git a/tests/ExcelReader.Tests/Reader/SourceParityTests.cs b/tests/ExcelReader.Tests/Reader/SourceParityTests.cs index d05a6c5f..dfebf8b7 100644 --- a/tests/ExcelReader.Tests/Reader/SourceParityTests.cs +++ b/tests/ExcelReader.Tests/Reader/SourceParityTests.cs @@ -1,6 +1,8 @@ using System.Globalization; using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Xlsb; using ExcelReader.Core.Reader.Xlsx; +using ExcelReader.Core.Writer.Xlsb; using ExcelReader.Core.Writer.Xlsx; namespace ExcelReader.Tests.Reader @@ -201,5 +203,92 @@ public void Xlsx_Cut_Short_Is_Invalid_Data() string path = WriteTemp(bytes[..(bytes.Length / 2)], ".xlsx"); Assert.Throws(() => Excel.FromXlsxFile(path)); } + + private static byte[] BuildXlsb() + { + using MemoryStream buffer = new(); + using (XlsbWorkbookWriter workbook = XlsbWorkbookWriter.Create(buffer, leaveOpen: true, new XlsbWriterOptions { UseSharedStrings = true })) + { + for (int s = 0; s < 3; s++) + { + XlsbSheetWriter sheet = workbook.AddSheet("sheet" + s); + for (int r = 0; r < 400; r++) + { + using XlsbRowWriter row = sheet.StartRow(); + row.Write("name-" + ((s * 1000) + (r % 50)).ToString(CultureInfo.InvariantCulture)); + row.Write((s * 100000) + r); + row.Write(r % 2 == 0); + } + sheet.End(); + sheet.Dispose(); + } + workbook.End(); + } + return buffer.ToArray(); + } + + private XlsbReader OpenXlsb(Kind kind, byte[] bytes) + { + switch (kind) + { + case Kind.Memory: + return Excel.FromXlsb(bytes); + case Kind.ExposedMemoryStream: + MemoryStream exposed = new(); + exposed.Write(bytes); + exposed.Position = 0; + return Excel.FromXlsb(exposed, leaveOpen: false); + case Kind.OpaqueMemoryStream: + return Excel.FromXlsb(new MemoryStream(bytes, writable: false), leaveOpen: false); + case Kind.Trickle: + return Excel.FromXlsb(new TrickleStream(bytes), leaveOpen: false); + case Kind.FileStream: + return Excel.FromXlsb(File.OpenRead(WriteTemp(bytes, ".xlsb")), leaveOpen: false); + case Kind.FilePath: + return Excel.FromXlsbFile(WriteTemp(bytes, ".xlsb")); + default: + return Excel.FromXlsb(new NonSeekableStream(bytes), leaveOpen: false); + } + } + + [Theory] + [MemberData(nameof(Kinds))] + public void Xlsb_Reads_The_Same_Rows_From_Every_Source(Kind kind) + { + byte[] bytes = BuildXlsb(); + using IExcelRowReader expected = Excel.FromXlsb(bytes); + using IExcelRowReader actual = OpenXlsb(kind, bytes); + Assert.Equal(Dump(expected), Dump(actual)); + } + + [Theory] + [MemberData(nameof(Kinds))] + public async Task Xlsb_Reads_The_Same_Rows_Asynchronously_From_Every_Source(Kind kind) + { + byte[] bytes = BuildXlsb(); + using IExcelRowReader expected = Excel.FromXlsb(bytes); + using IExcelRowReader actual = OpenXlsb(kind, bytes); + Assert.Equal(Dump(expected), await DumpAsync(actual)); + } + + [Fact] + public void Xlsb_File_Can_Be_Deleted_After_Dispose() + { + string path = WriteTemp(BuildXlsb(), ".xlsb"); + using (IExcelRowReader reader = Excel.FromXlsbFile(path)) + { + Assert.NotEmpty(Dump(reader)); + } + File.Delete(path); + Assert.False(File.Exists(path)); + } + + [Fact] + public void Xlsb_Cut_Short_Is_Invalid_Data() + { + byte[] bytes = BuildXlsb(); + string path = WriteTemp(bytes[..(bytes.Length / 2)], ".xlsb"); + Assert.Throws(() => Excel.FromXlsbFile(path)); + } } } From 38b7a100cf6d9a8e5ad85d3f492b9cd52f0d249d Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Tue, 6 Oct 2026 01:37:29 -0300 Subject: [PATCH 09/28] refactor(core): detect ZIP workbooks with ZipIndex and drop ZipArchive from the read path Excel.cs detects and dispatches with a ZipIndex; the Reopen* helpers, ZipEntryBytes, ZipReaderOpen and the ZipArchive helpers in WorkbookLookups are gone. Tests: deleted ZipIndexTests.ZipEntryBytesReadThrowsInvalidDataWhenEntryUnderDelivers (sole subject was ZipEntryBytes); rewrote the ReadViaZipArchive helper in ZipIndexTests to read the entry through ZipArchive directly. Co-Authored-By: Claude Sonnet 5.5 --- src/ExcelReader.Core/Reader/Excel.cs | 143 +++++------------- .../Reader/Internal/WorkbookLookups.cs | 22 --- .../Reader/Zip/ZipEntryBytes.cs | 121 --------------- .../Reader/Zip/ZipReaderOpen.cs | 81 ---------- .../Reader/SourceParityTests.cs | 79 ++++++++++ .../Reader/Zip/ZipIndexTests.cs | 21 +-- 6 files changed, 122 insertions(+), 345 deletions(-) delete mode 100644 src/ExcelReader.Core/Reader/Zip/ZipEntryBytes.cs delete mode 100644 src/ExcelReader.Core/Reader/Zip/ZipReaderOpen.cs diff --git a/src/ExcelReader.Core/Reader/Excel.cs b/src/ExcelReader.Core/Reader/Excel.cs index 59b767f8..b668631f 100644 --- a/src/ExcelReader.Core/Reader/Excel.cs +++ b/src/ExcelReader.Core/Reader/Excel.cs @@ -1,11 +1,11 @@ using System.Buffers; using System.Diagnostics.CodeAnalysis; -using System.IO.Compression; using System.Runtime.CompilerServices; using ExcelReader.Core.Crypto; using ExcelReader.Core.Parser; using ExcelReader.Core.Reader.Csv; using ExcelReader.Core.Reader.Schema; +using ExcelReader.Core.Reader.Sources; using ExcelReader.Core.Reader.Xls; using ExcelReader.Core.Reader.Xlsb; using ExcelReader.Core.Reader.Xlsx; @@ -44,7 +44,7 @@ public static XlsxReader FromXlsx(Stream stream, bool leaveOpen = true, ExcelRea /// /// Opens an XLSX workbook directly from an in-memory buffer. Reads the ZIP - /// central directory and decompresses parts without a + /// central directory and decompresses parts without a ZipArchive /// or intermediate — every part is fully materialized up front, so the returned /// reader never suspends, even under await foreach. /// @@ -109,7 +109,7 @@ public static XlsbReader FromXlsb(Stream stream, bool leaveOpen = true, ExcelRea /// /// Opens an XLSB workbook directly from an in-memory buffer. Reads the ZIP - /// central directory and decompresses parts without a + /// central directory and decompresses parts without a ZipArchive /// or intermediate — every part is fully materialized up front, so the returned /// reader never suspends, even under await foreach. /// @@ -352,7 +352,7 @@ public static IExcelRowReader Open(Stream stream, bool leaveOpen = true, ExcelRe /// /// Opens a workbook from an in-memory buffer, auto-detecting its format (XLSX/XLSB/XLS) from its /// signature. XLSX/XLSB route through instead of - /// a /, so the returned reader never + /// a ZipArchive/, so the returned reader never /// suspends, even under await foreach. /// /// The whole workbook file's bytes. Must outlive the returned reader. @@ -414,7 +414,7 @@ private static ExcelFileFormat ClassifyMemory(ReadOnlyMemory data, ExcelRe : ExcelFileFormat.Xls; } memZip = ZipIndex.Create(data, options); - return memZip.TryGetEntry("xl/workbook.bin"u8, out _) ? ExcelFileFormat.Xlsb : ExcelFileFormat.Xlsx; + return ClassifyZip(memZip); } /// @@ -629,7 +629,7 @@ private static void RequireOpenableFormat(ExcelFileFormat format) public static ExcelFileFormat DetectFileFormat(Stream stream) { ArgumentNullException.ThrowIfNull(stream); - ExcelFileFormat format = DetectSeekable(stream, out ZipArchive? zip); + ExcelFileFormat format = DetectSeekable(stream, leaveOpen: true, ExcelReaderOptions.Default, out ZipIndex? zip); zip?.Dispose(); return format; } @@ -654,25 +654,22 @@ public static ExcelFileFormat DetectFileFormat(ReadOnlyMemory data) public static async ValueTask DetectFileFormatAsync(Stream stream, CancellationToken ct = default) { ArgumentNullException.ThrowIfNull(stream); - (ExcelFileFormat format, ZipArchive? zip) = await DetectSeekableAsync(stream, ct).ConfigureAwait(false); - if (zip is not null) - { - await zip.DisposeAsync().ConfigureAwait(false); - } + (ExcelFileFormat format, ZipIndex? zip) = await DetectSeekableAsync(stream, leaveOpen: true, ExcelReaderOptions.Default, ct).ConfigureAwait(false); + zip?.Dispose(); return format; } private static IExcelRowReader OpenSeekable(Stream stream, bool leaveOpen, ExcelReaderOptions? options) { + ExcelReaderOptions effective = options ?? ExcelReaderOptions.Default; ExcelFileFormat format; - ZipArchive? zip = null; + ZipIndex? zip; try { - format = DetectSeekable(stream, out zip); + format = DetectSeekable(stream, leaveOpen, effective, out zip); } catch { - zip?.Dispose(); DisposeOnFailure(stream, leaveOpen); throw; } @@ -682,55 +679,37 @@ private static IExcelRowReader OpenSeekable(Stream stream, bool leaveOpen, Excel } if (format is ExcelFileFormat.EncryptedOoxml) { - ExcelReaderOptions effective = options ?? ExcelReaderOptions.Default; Stream decrypted = EncryptedPackageOpener.Decrypt(stream, leaveOpen, effective); return OpenDecryptedZip(decrypted, effective); } return format switch { ExcelFileFormat.Xls => new XlsReader(stream, leaveOpen, options), - ExcelFileFormat.Xlsb => ReopenXlsb(stream, leaveOpen, zip!, options), - ExcelFileFormat.Xlsx => ReopenXlsx(stream, leaveOpen, zip!, options), + ExcelFileFormat.Xlsb => XlsbReader.CreateFromIndex(zip!, effective), + ExcelFileFormat.Xlsx => XlsxReader.CreateFromIndex(zip!, effective), _ => throw new System.Diagnostics.UnreachableException(), }; } private static IExcelRowReader OpenDecryptedZip(Stream decrypted, ExcelReaderOptions options) { - ZipArchive? zip = null; - try - { - ExcelFileFormat zipFormat = ClassifyZipStream(decrypted, start: 0, out ZipArchive zipPeek); - zip = zipPeek; - return zipFormat switch - { - ExcelFileFormat.Xlsb => ReopenXlsb(decrypted, leaveOpen: false, zip, options), - ExcelFileFormat.Xlsx => ReopenXlsx(decrypted, leaveOpen: false, zip, options), - _ => throw new System.Diagnostics.UnreachableException(), - }; - } - catch - { - zip?.Dispose(); - decrypted.Dispose(); - throw; - } + ZipIndex zip = ZipIndex.Create(ByteSource.FromStream(decrypted, leaveOpen: false), options); + return ClassifyZip(zip) is ExcelFileFormat.Xlsb + ? XlsbReader.CreateFromIndex(zip, options) + : XlsxReader.CreateFromIndex(zip, options); } private static async ValueTask OpenSeekableAsync(Stream stream, bool leaveOpen, ExcelReaderOptions? options, CancellationToken ct) { + ExcelReaderOptions effective = options ?? ExcelReaderOptions.Default; ExcelFileFormat format; - ZipArchive? zip = null; + ZipIndex? zip; try { - (format, zip) = await DetectSeekableAsync(stream, ct).ConfigureAwait(false); + (format, zip) = await DetectSeekableAsync(stream, leaveOpen, effective, ct).ConfigureAwait(false); } catch { - if (zip is not null) - { - await zip.DisposeAsync().ConfigureAwait(false); - } await DisposeOnFailureAsync(stream, leaveOpen).ConfigureAwait(false); throw; } @@ -741,42 +720,25 @@ private static async ValueTask OpenSeekableAsync(Stream stream, } if (format is ExcelFileFormat.EncryptedOoxml) { - ExcelReaderOptions effective = options ?? ExcelReaderOptions.Default; Stream decrypted = EncryptedPackageOpener.Decrypt(stream, leaveOpen, effective); return await OpenDecryptedZipAsync(decrypted, effective, ct).ConfigureAwait(false); } return format switch { ExcelFileFormat.Xls => await XlsReader.CreateAsync(stream, leaveOpen, options, ct).ConfigureAwait(false), - ExcelFileFormat.Xlsb => await ReopenXlsbAsync(stream, leaveOpen, zip!, options, ct).ConfigureAwait(false), - ExcelFileFormat.Xlsx => await ReopenXlsxAsync(stream, leaveOpen, zip!, options, ct).ConfigureAwait(false), + ExcelFileFormat.Xlsb => await XlsbReader.CreateFromIndexAsync(zip!, effective, ct).ConfigureAwait(false), + ExcelFileFormat.Xlsx => await XlsxReader.CreateFromIndexAsync(zip!, effective, ct).ConfigureAwait(false), _ => throw new System.Diagnostics.UnreachableException(), }; } private static async ValueTask OpenDecryptedZipAsync(Stream decrypted, ExcelReaderOptions options, CancellationToken ct) { - ZipArchive? zip = null; - try - { - ExcelFileFormat zipFormat = ClassifyZipStream(decrypted, start: 0, out ZipArchive zipPeek); - zip = zipPeek; - return zipFormat switch - { - ExcelFileFormat.Xlsb => await ReopenXlsbAsync(decrypted, leaveOpen: false, zip, options, ct).ConfigureAwait(false), - ExcelFileFormat.Xlsx => await ReopenXlsxAsync(decrypted, leaveOpen: false, zip, options, ct).ConfigureAwait(false), - _ => throw new System.Diagnostics.UnreachableException(), - }; - } - catch - { - if (zip is not null) - { - await zip.DisposeAsync().ConfigureAwait(false); - } - await decrypted.DisposeAsync().ConfigureAwait(false); - throw; - } + ByteSource source = await ByteSource.FromStreamAsync(decrypted, leaveOpen: false, ct).ConfigureAwait(false); + ZipIndex zip = await ZipIndex.CreateAsync(source, options, ct).ConfigureAwait(false); + return ClassifyZip(zip) is ExcelFileFormat.Xlsb + ? await XlsbReader.CreateFromIndexAsync(zip, options, ct).ConfigureAwait(false) + : await XlsxReader.CreateFromIndexAsync(zip, options, ct).ConfigureAwait(false); } private static void DisposeOnFailure(Stream stream, bool leaveOpen) @@ -816,43 +778,13 @@ private static bool TryClassifyHeader(ReadOnlySpan sig, out ExcelFileForma return true; } - private static XlsxReader ReopenXlsx(Stream stream, bool leaveOpen, ZipArchive peeked, ExcelReaderOptions? options) + private static ExcelFileFormat ClassifyZip(ZipIndex zip) { - peeked.Dispose(); - return new XlsxReader(stream, leaveOpen, options); - } - - private static async ValueTask ReopenXlsxAsync( - Stream stream, bool leaveOpen, ZipArchive peeked, ExcelReaderOptions? options, CancellationToken ct) - { - await peeked.DisposeAsync().ConfigureAwait(false); - return await XlsxReader.CreateAsync(stream, leaveOpen, options, ct).ConfigureAwait(false); - } - - private static XlsbReader ReopenXlsb(Stream stream, bool leaveOpen, ZipArchive peeked, ExcelReaderOptions? options) - { - peeked.Dispose(); - return new XlsbReader(stream, leaveOpen, options); - } - - private static async ValueTask ReopenXlsbAsync( - Stream stream, bool leaveOpen, ZipArchive peeked, ExcelReaderOptions? options, CancellationToken ct) - { - await peeked.DisposeAsync().ConfigureAwait(false); - return await XlsbReader.CreateAsync(stream, leaveOpen, options, ct).ConfigureAwait(false); - } - - private static ExcelFileFormat ClassifyZipStream(Stream stream, long start, out ZipArchive zip) - { - var zipPeek = new ZipArchive(stream, ZipArchiveMode.Read, leaveOpen: true); - zip = zipPeek; - bool isXlsb = zipPeek.GetEntry("xl/workbook.bin") is not null; - stream.Position = start; - return isXlsb ? ExcelFileFormat.Xlsb : ExcelFileFormat.Xlsx; + return zip.TryGetEntry("xl/workbook.bin"u8, out _) ? ExcelFileFormat.Xlsb : ExcelFileFormat.Xlsx; } [SkipLocalsInit] - private static ExcelFileFormat DetectSeekable(Stream stream, out ZipArchive? zip) + private static ExcelFileFormat DetectSeekable(Stream stream, bool leaveOpen, ExcelReaderOptions options, out ZipIndex? zip) { zip = null; RequireSeekable(stream); @@ -871,12 +803,13 @@ private static ExcelFileFormat DetectSeekable(Stream stream, out ZipArchive? zip ? ExcelFileFormat.EncryptedOoxml : ExcelFileFormat.Xls; } - ExcelFileFormat zipFormat = ClassifyZipStream(stream, start, out ZipArchive zipPeek); - zip = zipPeek; - return zipFormat; + zip = ZipIndex.Create(ByteSource.FromStream(stream, leaveOpen), options); + stream.Position = start; + return ClassifyZip(zip); } - private static async ValueTask<(ExcelFileFormat Format, ZipArchive? Zip)> DetectSeekableAsync(Stream stream, CancellationToken ct) + private static async ValueTask<(ExcelFileFormat Format, ZipIndex? Zip)> DetectSeekableAsync( + Stream stream, bool leaveOpen, ExcelReaderOptions options, CancellationToken ct) { RequireSeekable(stream); long start = stream.Position; @@ -905,8 +838,10 @@ private static ExcelFileFormat DetectSeekable(Stream stream, out ZipArchive? zip : ExcelFileFormat.Xls; return (cfbFormat, null); } - ExcelFileFormat zipFormat = ClassifyZipStream(stream, start, out ZipArchive zip); - return (zipFormat, zip); + ByteSource source = await ByteSource.FromStreamAsync(stream, leaveOpen, ct).ConfigureAwait(false); + ZipIndex zip = await ZipIndex.CreateAsync(source, options, ct).ConfigureAwait(false); + stream.Position = start; + return (ClassifyZip(zip), zip); } private static bool TryDecryptCfbStream(Stream stream, bool leaveOpen, ExcelReaderOptions? options, out Stream decrypted) diff --git a/src/ExcelReader.Core/Reader/Internal/WorkbookLookups.cs b/src/ExcelReader.Core/Reader/Internal/WorkbookLookups.cs index d4290602..186910ae 100644 --- a/src/ExcelReader.Core/Reader/Internal/WorkbookLookups.cs +++ b/src/ExcelReader.Core/Reader/Internal/WorkbookLookups.cs @@ -1,4 +1,3 @@ -using System.IO.Compression; using System.Runtime.CompilerServices; using System.Text; using ExcelReader.Core.Reader.Zip; @@ -66,12 +65,6 @@ internal static (int Start, int Length, int ValidIndex) SharedAt(int[] sharedOff return (sharedOffsets[index], sharedOffsets[index + 1] - sharedOffsets[index], index); } - internal static ZipArchiveEntry GetWorksheetEntry(ZipArchive zip, string path) - { - return zip.GetEntry(path) - ?? throw new InvalidDataException($"Worksheet part not found: {path}"); - } - [SkipLocalsInit] internal static ZipEntryRef GetWorksheetEntry(ZipIndex memZip, string path) { @@ -84,21 +77,6 @@ internal static ZipEntryRef GetWorksheetEntry(ZipIndex memZip, string path) : throw new InvalidDataException($"Worksheet part not found: {path}"); } - internal static LimitedReadStream OpenEntryStream( - ZipArchiveEntry entry, DecompressedByteCounter counter, ExcelReaderOptions options, - string entryLimitName = "", long entryLimit = 0) - { - return Wrap(entry.Open(), counter, options, entryLimitName, entryLimit, entry.Length); - } - - internal static async ValueTask OpenEntryStreamAsync( - ZipArchiveEntry entry, DecompressedByteCounter counter, ExcelReaderOptions options, - CancellationToken ct, string entryLimitName = "", long entryLimit = 0) - { - Stream opened = await entry.OpenAsync(ct).ConfigureAwait(false); - return Wrap(opened, counter, options, entryLimitName, entryLimit, entry.Length); - } - private const long PrefetchMinUncompressedSize = 256 * 1024; internal static LimitedReadStream Wrap( diff --git a/src/ExcelReader.Core/Reader/Zip/ZipEntryBytes.cs b/src/ExcelReader.Core/Reader/Zip/ZipEntryBytes.cs deleted file mode 100644 index 270c9557..00000000 --- a/src/ExcelReader.Core/Reader/Zip/ZipEntryBytes.cs +++ /dev/null @@ -1,121 +0,0 @@ -using System.Buffers; -using System.IO.Compression; -using ExcelReader.Core.Reader.Internal; - -namespace ExcelReader.Core.Reader.Zip -{ - internal static class ZipEntryBytes - { - private static void ThrowIfEntryLengthExceedsLimits( - ZipArchiveEntry entry, DecompressedByteCounter counter, string entryLimitName, long entryLimit) - { - LimitChecks.ThrowIfEntryLengthExceeds(entry.Length, counter.Remaining, - nameof(ExcelReaderOptions.MaxTotalDecompressedBytes)); - if (entryLimit > 0) - { - LimitChecks.ThrowIfEntryLengthExceeds(entry.Length, entryLimit, entryLimitName); - } - } - - internal static ZipPart Read( - ZipArchive zip, - string name, - DecompressedByteCounter counter, - string entryLimitName = "", - long entryLimit = 0) - { - ZipArchiveEntry? entry = zip.GetEntry(name); - return entry is null ? default : Read(entry, counter, entryLimitName, entryLimit); - } - - internal static ZipPart Read( - ZipArchiveEntry entry, - DecompressedByteCounter counter, - string entryLimitName = "", - long entryLimit = 0) - { - ThrowIfEntryLengthExceedsLimits(entry, counter, entryLimitName, entryLimit); - using var stream = new LimitedReadStream(entry.Open(), counter, entryLimitName, entryLimit); - int length = checked((int)entry.Length); - byte[] rented = ArrayPool.Shared.Rent(length); - try - { - ReadExactlyChecked(stream, rented.AsSpan(0, length)); - } - catch - { - ArrayPool.Shared.Return(rented); - throw; - } - return new ZipPart(rented.AsMemory(0, length), rented); - } - - private static void ReadExactlyChecked(Stream stream, Span destination) - { - try - { - stream.ReadExactly(destination); - } - catch (EndOfStreamException ex) - { - throw new InvalidDataException("The ZIP entry produced less data than its declared uncompressed size.", ex); - } - } - - internal static async ValueTask ReadAsync( - ZipArchive zip, - string name, - DecompressedByteCounter counter, - CancellationToken ct, - string entryLimitName = "", - long entryLimit = 0) - { - ZipArchiveEntry? entry = zip.GetEntry(name); - if (entry is null) - { - return default; - } - return await ReadAsync(entry, counter, ct, entryLimitName, entryLimit).ConfigureAwait(false); - } - - internal static async ValueTask ReadAsync( - ZipArchiveEntry entry, - DecompressedByteCounter counter, - CancellationToken ct, - string entryLimitName = "", - long entryLimit = 0) - { - ThrowIfEntryLengthExceedsLimits(entry, counter, entryLimitName, entryLimit); - Stream opened = await entry.OpenAsync(ct).ConfigureAwait(false); - var stream = new LimitedReadStream(opened, counter, entryLimitName, entryLimit); - await using (stream.ConfigureAwait(false)) - { - int length = checked((int)entry.Length); - byte[] rented = ArrayPool.Shared.Rent(length); - try - { - await ReadExactlyCheckedAsync(stream, rented.AsMemory(0, length), ct).ConfigureAwait(false); - } - catch - { - ArrayPool.Shared.Return(rented); - throw; - } - return new ZipPart(rented.AsMemory(0, length), rented); - } - } - - private static async ValueTask ReadExactlyCheckedAsync(Stream stream, Memory destination, CancellationToken ct) - { - try - { - await stream.ReadExactlyAsync(destination, ct).ConfigureAwait(false); - } - catch (EndOfStreamException ex) - { - throw new InvalidDataException("The ZIP entry produced less data than its declared uncompressed size.", ex); - } - } - - } -} diff --git a/src/ExcelReader.Core/Reader/Zip/ZipReaderOpen.cs b/src/ExcelReader.Core/Reader/Zip/ZipReaderOpen.cs deleted file mode 100644 index 3a08afc1..00000000 --- a/src/ExcelReader.Core/Reader/Zip/ZipReaderOpen.cs +++ /dev/null @@ -1,81 +0,0 @@ -using System.IO.Compression; -using ExcelReader.Core.Reader.Internal; - -namespace ExcelReader.Core.Reader.Zip -{ - internal static class ZipReaderOpen - { - internal static ZipArchive Open(Stream stream, bool leaveOpen) - { - try - { - return new ZipArchive(stream, ZipArchiveMode.Read, leaveOpen: true); - } - catch - { - if (!leaveOpen) - { - stream.Dispose(); - } - throw; - } - } - - internal static async ValueTask OpenAsync( - Stream stream, bool leaveOpen, ExcelReaderOptions options, Func> parseBody, CancellationToken ct = default) - { - ZipArchive? zip = null; - try - { - zip = await ZipArchive.CreateAsync(stream, ZipArchiveMode.Read, leaveOpen: true, entryNameEncoding: null, ct).ConfigureAwait(false); - LimitChecks.ThrowIfTooManyEntries(zip.Entries.Count, options); - return await parseBody(zip).ConfigureAwait(false); - } - catch - { - if (zip is not null) - { - await zip.DisposeAsync().ConfigureAwait(false); - } - if (!leaveOpen) - { - await stream.DisposeAsync().ConfigureAwait(false); - } - throw; - } - } - - internal static async ValueTask FromOpenZipAsync( - Stream stream, bool leaveOpen, ZipArchive zip, ExcelReaderOptions options, - Func> parseBody) - { - try - { - LimitChecks.ThrowIfTooManyEntries(zip.Entries.Count, options); - return await parseBody(zip).ConfigureAwait(false); - } - catch - { - await zip.DisposeAsync().ConfigureAwait(false); - if (!leaveOpen) - { - await stream.DisposeAsync().ConfigureAwait(false); - } - throw; - } - } - - internal static TResult FromMemory(ZipIndex memZip, Func build) - { - try - { - return build(memZip); - } - catch - { - memZip.Dispose(); - throw; - } - } - } -} diff --git a/tests/ExcelReader.Tests/Reader/SourceParityTests.cs b/tests/ExcelReader.Tests/Reader/SourceParityTests.cs index dfebf8b7..21573e89 100644 --- a/tests/ExcelReader.Tests/Reader/SourceParityTests.cs +++ b/tests/ExcelReader.Tests/Reader/SourceParityTests.cs @@ -290,5 +290,84 @@ public void Xlsb_Cut_Short_Is_Invalid_Data() string path = WriteTemp(bytes[..(bytes.Length / 2)], ".xlsb"); Assert.Throws(() => Excel.FromXlsbFile(path)); } + + public static TheoryData SeekableStreamKinds => + [ + Kind.ExposedMemoryStream, Kind.OpaqueMemoryStream, Kind.Trickle, Kind.FileStream, + ]; + + private Stream OpenRawStream(Kind kind, byte[] bytes) + { + switch (kind) + { + case Kind.ExposedMemoryStream: + MemoryStream exposed = new(); + exposed.Write(bytes); + exposed.Position = 0; + return exposed; + case Kind.OpaqueMemoryStream: + return new MemoryStream(bytes, writable: false); + case Kind.Trickle: + return new TrickleStream(bytes); + default: + return File.OpenRead(WriteTemp(bytes, ".bin")); + } + } + + [Theory] + [MemberData(nameof(SeekableStreamKinds))] + public void DetectFileFormat_Leaves_The_Stream_Open_At_Its_Position(Kind kind) + { + using Stream xlsx = OpenRawStream(kind, BuildXlsx()); + Assert.Equal(ExcelFileFormat.Xlsx, Excel.DetectFileFormat(xlsx)); + Assert.Equal(0, xlsx.Position); + Assert.True(xlsx.CanRead); + + using Stream xlsb = OpenRawStream(kind, BuildXlsb()); + Assert.Equal(ExcelFileFormat.Xlsb, Excel.DetectFileFormat(xlsb)); + Assert.Equal(0, xlsb.Position); + Assert.True(xlsb.CanRead); + } + + [Theory] + [MemberData(nameof(SeekableStreamKinds))] + public async Task DetectFileFormatAsync_Leaves_The_Stream_Open_At_Its_Position(Kind kind) + { + using Stream xlsx = OpenRawStream(kind, BuildXlsx()); + Assert.Equal(ExcelFileFormat.Xlsx, await Excel.DetectFileFormatAsync(xlsx, TestContext.Current.CancellationToken)); + Assert.Equal(0, xlsx.Position); + Assert.True(xlsx.CanRead); + } + + [Theory] + [MemberData(nameof(SeekableStreamKinds))] + public void Open_Detects_And_Reads_Both_Zip_Formats(Kind kind) + { + byte[] xlsx = BuildXlsx(); + using IExcelRowReader expectedXlsx = Excel.FromXlsx(xlsx); + using IExcelRowReader actualXlsx = Excel.Open(OpenRawStream(kind, xlsx), leaveOpen: false); + Assert.Equal(Dump(expectedXlsx), Dump(actualXlsx)); + + byte[] xlsb = BuildXlsb(); + using IExcelRowReader expectedXlsb = Excel.FromXlsb(xlsb); + using IExcelRowReader actualXlsb = Excel.Open(OpenRawStream(kind, xlsb), leaveOpen: false); + Assert.Equal(Dump(expectedXlsb), Dump(actualXlsb)); + } + + [Theory] + [MemberData(nameof(SeekableStreamKinds))] + public async Task OpenAsync_Detects_And_Reads_Both_Zip_Formats(Kind kind) + { + CancellationToken ct = TestContext.Current.CancellationToken; + byte[] xlsx = BuildXlsx(); + using IExcelRowReader expectedXlsx = Excel.FromXlsx(xlsx); + using IExcelRowReader actualXlsx = await Excel.OpenAsync(OpenRawStream(kind, xlsx), leaveOpen: false, ct: ct); + Assert.Equal(Dump(expectedXlsx), await DumpAsync(actualXlsx)); + + byte[] xlsb = BuildXlsb(); + using IExcelRowReader expectedXlsb = Excel.FromXlsb(xlsb); + using IExcelRowReader actualXlsb = await Excel.OpenAsync(OpenRawStream(kind, xlsb), leaveOpen: false, ct: ct); + Assert.Equal(Dump(expectedXlsb), await DumpAsync(actualXlsb)); + } } } diff --git a/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs index 3d2dea3c..8903de08 100644 --- a/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs +++ b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs @@ -588,21 +588,6 @@ private static uint Crc32(ReadOnlySpan data) return ~crc; } - [Fact] - public void ZipEntryBytesReadThrowsInvalidDataWhenEntryUnderDelivers() - { - using MemoryStream built = WorkbookBuilder.Build("""1"""); - byte[] zipBytes = built.ToArray(); - uint realLength = ReadDeclaredUncompressedSize(zipBytes, "xl/workbook.xml"); - PatchCentralDirectoryUInt32(zipBytes, "xl/workbook.xml", fieldOffset: 24, realLength + 64); - - using var ms = new MemoryStream(zipBytes); - using var zip = new ZipArchive(ms, ZipArchiveMode.Read); - InvalidDataException ex = Assert.Throws( - () => ZipEntryBytes.Read(zip, "xl/workbook.xml", new DecompressedByteCounter(0))); - Assert.Contains("less data", ex.Message, StringComparison.OrdinalIgnoreCase); - } - [Fact] public void ZipArchiveEntryOpenSilentlyTruncatesAtDeclaredLengthUnderOverDelivery() { @@ -631,8 +616,10 @@ private static byte[] ReadViaZipArchive(byte[] zipBytes, string entryName) { using var ms = new MemoryStream(zipBytes); using var zip = new ZipArchive(ms, ZipArchiveMode.Read); - using ZipPart part = ZipEntryBytes.Read(zip, entryName, new DecompressedByteCounter(0)); - return part.Memory.ToArray(); + using Stream entry = zip.GetEntry(entryName)!.Open(); + using var copy = new MemoryStream(); + entry.CopyTo(copy); + return copy.ToArray(); } private static byte[] BuildZipWithOneEntry(string entryName, byte[] payload, CompressionLevel level) From fc38fdf00c66f11db1045a6425b39968eddcd2a7 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Tue, 6 Oct 2026 01:42:00 -0300 Subject: [PATCH 10/28] refactor(core): load XLS sectors through ByteSource instead of seeking a shared stream Co-Authored-By: Claude Haiku 4.5 --- .../Reader/Xls/WorkbookStream.cs | 23 ++--- .../Reader/Xls/XlsCompoundFile.cs | 3 +- .../Reader/SourceParityTests.cs | 92 +++++++++++++++++++ 3 files changed, 103 insertions(+), 15 deletions(-) diff --git a/src/ExcelReader.Core/Reader/Xls/WorkbookStream.cs b/src/ExcelReader.Core/Reader/Xls/WorkbookStream.cs index 42c430b5..78ed0d7a 100644 --- a/src/ExcelReader.Core/Reader/Xls/WorkbookStream.cs +++ b/src/ExcelReader.Core/Reader/Xls/WorkbookStream.cs @@ -1,6 +1,7 @@ using System.Buffers; using System.Diagnostics.CodeAnalysis; using System.Runtime.InteropServices; +using ExcelReader.Core.Reader.Sources; namespace ExcelReader.Core.Reader.Xls { @@ -9,8 +10,7 @@ internal sealed class WorkbookStream : IDisposable { private const int HeaderSize = 512; - private readonly Stream? _source; - private readonly bool _ownsSource; + private readonly ByteSource? _source; private readonly int[] _chain; private readonly int _chainLength; private bool _chainReturned; @@ -29,10 +29,9 @@ internal enum SourceKind internal SourceKind Kind { get; } - private WorkbookStream(Stream? source, bool ownsSource, int[] chain, int chainLength, ReadOnlyMemory memory, int sectorSize, long length, SourceKind kind) + private WorkbookStream(ByteSource? source, int[] chain, int chainLength, ReadOnlyMemory memory, int sectorSize, long length, SourceKind kind) { _source = source; - _ownsSource = ownsSource; _chain = chain; _chainLength = chainLength; SectorSize = sectorSize; @@ -54,19 +53,19 @@ private WorkbookStream(Stream? source, bool ownsSource, int[] chain, int chainLe _fileLength = Buffer.Length; } - internal static WorkbookStream Streamed(Stream source, bool ownsSource, int[] chain, int chainLength, int sectorSize, long length) + internal static WorkbookStream Streamed(ByteSource source, int[] chain, int chainLength, int sectorSize, long length) { - return new WorkbookStream(source, ownsSource, chain, chainLength, default, sectorSize, length, SourceKind.Streamed); + return new WorkbookStream(source, chain, chainLength, default, sectorSize, length, SourceKind.Streamed); } internal static WorkbookStream InMemory(ReadOnlyMemory data) { - return new WorkbookStream(null, ownsSource: false, [], 0, data, sectorSize: 1, data.Length, SourceKind.Contiguous); + return new WorkbookStream(null, [], 0, data, sectorSize: 1, data.Length, SourceKind.Contiguous); } internal static WorkbookStream Chained(ReadOnlyMemory data, int[] chain, int chainLength, int sectorSize, long length) { - return new WorkbookStream(null, ownsSource: false, chain, chainLength, data, sectorSize, length, SourceKind.Chained); + return new WorkbookStream(null, chain, chainLength, data, sectorSize, length, SourceKind.Chained); } internal BiffCursor OpenCursor() @@ -156,17 +155,13 @@ internal int LoadSectors(int chainIndex, Span dest) count++; } long offset = HeaderSize + ((long)_chain[chainIndex] * SectorSize); - _source!.Seek(offset, SeekOrigin.Begin); - _source.ReadExactly(dest[..(count * SectorSize)]); + _source!.ReadExactly(offset, dest[..(count * SectorSize)]); return count; } public void Dispose() { - if (_ownsSource) - { - _source?.Dispose(); - } + _source?.Dispose(); if (_chainLength > 0 && !_chainReturned) { _chainReturned = true; diff --git a/src/ExcelReader.Core/Reader/Xls/XlsCompoundFile.cs b/src/ExcelReader.Core/Reader/Xls/XlsCompoundFile.cs index 670170f1..c7771f65 100644 --- a/src/ExcelReader.Core/Reader/Xls/XlsCompoundFile.cs +++ b/src/ExcelReader.Core/Reader/Xls/XlsCompoundFile.cs @@ -2,6 +2,7 @@ using System.Runtime.InteropServices; using ExcelReader.Core.Crypto; using ExcelReader.Core.Reader.Internal; +using ExcelReader.Core.Reader.Sources; namespace ExcelReader.Core.Reader.Xls { @@ -138,7 +139,7 @@ private static WorkbookStream BuildWorkbook(Stream source, bool ownsSource, Exce { return WorkbookStream.Chained(memory, chain, chainCount, cfb.SectorSize, workbook.Size); } - return WorkbookStream.Streamed(source, ownsSource, chain, chainCount, cfb.SectorSize, workbook.Size); + return WorkbookStream.Streamed(ByteSource.FromStream(source, leaveOpen: !ownsSource), chain, chainCount, cfb.SectorSize, workbook.Size); } finally { diff --git a/tests/ExcelReader.Tests/Reader/SourceParityTests.cs b/tests/ExcelReader.Tests/Reader/SourceParityTests.cs index 21573e89..b0b9f176 100644 --- a/tests/ExcelReader.Tests/Reader/SourceParityTests.cs +++ b/tests/ExcelReader.Tests/Reader/SourceParityTests.cs @@ -1,7 +1,9 @@ using System.Globalization; using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Xls; using ExcelReader.Core.Reader.Xlsb; using ExcelReader.Core.Reader.Xlsx; +using ExcelReader.Core.Writer.Xls; using ExcelReader.Core.Writer.Xlsb; using ExcelReader.Core.Writer.Xlsx; @@ -204,6 +206,96 @@ public void Xlsx_Cut_Short_Is_Invalid_Data() Assert.Throws(() => Excel.FromXlsxFile(path)); } + private static byte[] BuildXls() + { + using MemoryStream buffer = new(); + using (XlsWorkbookWriter workbook = XlsWorkbookWriter.Create(buffer, leaveOpen: true)) + { + for (int s = 0; s < 3; s++) + { + XlsSheetWriter sheet = workbook.AddSheet("sheet" + s); + for (int r = 0; r < 400; r++) + { + using XlsRowWriter row = sheet.StartRow(); + row.Write("name-" + ((s * 1000) + (r % 50)).ToString(CultureInfo.InvariantCulture)); + row.Write((s * 100000) + r); + row.Write(r % 2 == 0); + } + sheet.End(); + sheet.Dispose(); + } + workbook.End(); + } + return buffer.ToArray(); + } + + private XlsReader OpenXls(Kind kind, byte[] bytes) + { + switch (kind) + { + case Kind.Memory: + return Excel.FromXls(bytes); + case Kind.ExposedMemoryStream: + MemoryStream exposed = new(); + exposed.Write(bytes); + exposed.Position = 0; + return Excel.FromXls(exposed, leaveOpen: false); + case Kind.OpaqueMemoryStream: + return Excel.FromXls(new MemoryStream(bytes, writable: false), leaveOpen: false); + case Kind.Trickle: + return Excel.FromXls(new TrickleStream(bytes), leaveOpen: false); + case Kind.FileStream: + return Excel.FromXls(File.OpenRead(WriteTemp(bytes, ".xls")), leaveOpen: false); + case Kind.FilePath: + return Excel.FromXlsFile(WriteTemp(bytes, ".xls")); + default: + return Excel.FromXls(new NonSeekableStream(bytes), leaveOpen: false); + } + } + + [Theory] + [MemberData(nameof(Kinds))] + public void Xls_Reads_The_Same_Rows_From_Every_Source(Kind kind) + { + byte[] bytes = BuildXls(); + using IExcelRowReader expected = Excel.FromXls(bytes); + using IExcelRowReader actual = OpenXls(kind, bytes); + Assert.Equal(Dump(expected), Dump(actual)); + } + + [Theory] + [MemberData(nameof(Kinds))] + public void Xls_Interleaved_Enumerators_Keep_Their_Own_Position(Kind kind) + { + byte[] bytes = BuildXls(); + using IExcelRowReader reader = OpenXls(kind, bytes); + reader.MoveToSheet(0); + using IExcelRowEnumerator first = reader.GetEnumerator(); + reader.MoveToSheet(1); + using IExcelRowEnumerator second = reader.GetEnumerator(); + for (int r = 0; r < 400; r++) + { + Assert.True(first.MoveNext()); + Assert.True(second.MoveNext()); + Assert.Equal("name-" + (r % 50).ToString(CultureInfo.InvariantCulture), first.Current[0].GetString()); + Assert.Equal("name-" + (1000 + (r % 50)).ToString(CultureInfo.InvariantCulture), second.Current[0].GetString()); + } + Assert.False(first.MoveNext()); + Assert.False(second.MoveNext()); + } + + [Fact] + public void Xls_File_Can_Be_Deleted_After_Dispose() + { + string path = WriteTemp(BuildXls(), ".xls"); + using (IExcelRowReader reader = Excel.FromXlsFile(path)) + { + Assert.NotEmpty(Dump(reader)); + } + File.Delete(path); + Assert.False(File.Exists(path)); + } + private static byte[] BuildXlsb() { using MemoryStream buffer = new(); From 696134e49e479e5c3c40f8231d681d4e9959e603 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Tue, 6 Oct 2026 03:03:15 -0300 Subject: [PATCH 11/28] fix(core): accept bytes after the ZIP end-of-central-directory record Co-Authored-By: Claude Sonnet 5.5 --- src/ExcelReader.Core/Reader/Zip/ZipIndex.cs | 2 +- .../Reader/Zip/ZipIndexSourceTests.cs | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/ExcelReader.Core/Reader/Zip/ZipIndex.cs b/src/ExcelReader.Core/Reader/Zip/ZipIndex.cs index ec9ae6c5..498c6f74 100644 --- a/src/ExcelReader.Core/Reader/Zip/ZipIndex.cs +++ b/src/ExcelReader.Core/Reader/Zip/ZipIndex.cs @@ -576,7 +576,7 @@ private static long FindEocd(ReadOnlySpan span) continue; } int commentLength = BinaryPrimitives.ReadUInt16LittleEndian(window.Slice(index + 20, 2)); - if (index + EocdFixedSize + commentLength == window.Length) + if (index + EocdFixedSize + commentLength <= window.Length) { return windowStart + index; } diff --git a/tests/ExcelReader.Tests/Reader/Zip/ZipIndexSourceTests.cs b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexSourceTests.cs index ef6bf933..416ef1cc 100644 --- a/tests/ExcelReader.Tests/Reader/Zip/ZipIndexSourceTests.cs +++ b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexSourceTests.cs @@ -236,6 +236,27 @@ record = i; Assert.Throws(() => index.OpenEntryStream(entry, Counter(), ExcelReaderOptions.Default).Dispose()); } + [Theory] + [MemberData(nameof(Kinds))] + public void Bytes_After_The_End_Of_Central_Directory_Are_Ignored(Kind kind) + { + byte[] zipBytes = BuildZip(dataDescriptors: false); + foreach (int extra in (int[])[1, 100]) + { + byte[] padded = new byte[zipBytes.Length + extra]; + zipBytes.CopyTo(padded, 0); + padded.AsSpan(zipBytes.Length).Fill(0xAB); + + using ZipIndex index = ZipIndex.Create(Open(kind, padded), ExcelReaderOptions.Default); + for (int i = 0; i < Names.Length; i++) + { + Assert.True(index.TryGetEntry(System.Text.Encoding.UTF8.GetBytes(Names[i]), out ZipEntryRef entry)); + using ZipPart part = index.OpenPart(entry, Counter()); + Assert.True(part.Memory.Span.SequenceEqual(Content(i))); + } + } + } + [Fact] public void A_Failed_Create_Disposes_The_Source() { From f03340eaaec86005c903dd3748c5cf340f52b62d Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Tue, 6 Oct 2026 03:07:04 -0300 Subject: [PATCH 12/28] test(core): cover ZIP64, async and failure paths of the positional ZIP index ZIP64 fixtures now also run through a stream source, sync and async. The mutation differential test covers sync and async part and entry-stream reads. Entry streams are compared with the memory entry stream, since streams never verify declared sizes on any source. Reader entry points are checked for disposing the stream on failure, and the "less data than declared" guard is covered again. No product defect found. Co-Authored-By: Claude Sonnet 5.5 --- .../Reader/SourceParityTests.cs | 54 ++++ .../Reader/Zip/ZipIndexSourceTests.cs | 106 +++++++- .../Reader/Zip/ZipIndexTests.cs | 253 ++++++++++++++++++ 3 files changed, 405 insertions(+), 8 deletions(-) diff --git a/tests/ExcelReader.Tests/Reader/SourceParityTests.cs b/tests/ExcelReader.Tests/Reader/SourceParityTests.cs index b0b9f176..b1dea41c 100644 --- a/tests/ExcelReader.Tests/Reader/SourceParityTests.cs +++ b/tests/ExcelReader.Tests/Reader/SourceParityTests.cs @@ -461,5 +461,59 @@ public async Task OpenAsync_Detects_And_Reads_Both_Zip_Formats(Kind kind) using IExcelRowReader actualXlsb = await Excel.OpenAsync(OpenRawStream(kind, xlsb), leaveOpen: false, ct: ct); Assert.Equal(Dump(expectedXlsb), await DumpAsync(actualXlsb)); } + + private static byte[] CorruptZip() + { + byte[] bytes = new byte[200]; + bytes.AsSpan().Fill(0x41); + bytes[0] = 0x50; + bytes[1] = 0x4B; + bytes[2] = 0x03; + bytes[3] = 0x04; + return bytes; + } + + private static byte[] ZipWithoutWorkbook() + { + using MemoryStream buffer = new(); + using (System.IO.Compression.ZipArchive zip = new(buffer, System.IO.Compression.ZipArchiveMode.Create, leaveOpen: true)) + { + using StreamWriter writer = new(zip.CreateEntry("hello.txt").Open()); + writer.Write("hello"); + } + return buffer.ToArray(); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task A_Corrupt_Zip_Opened_Through_Every_Entry_Point_Honours_LeaveOpen(bool leaveOpen) + { + CancellationToken ct = TestContext.Current.CancellationToken; + byte[] bytes = CorruptZip(); + TrickleStream open = new(bytes); + Assert.Throws(() => Excel.Open(open, leaveOpen)); + Assert.Equal(leaveOpen, open.CanRead); + + TrickleStream openAsync = new(bytes); + await Assert.ThrowsAsync(async () => await Excel.OpenAsync(openAsync, leaveOpen, ct: ct)); + Assert.Equal(leaveOpen, openAsync.CanRead); + + TrickleStream xlsx = new(bytes); + Assert.Throws(() => Excel.FromXlsx(xlsx, leaveOpen)); + Assert.Equal(leaveOpen, xlsx.CanRead); + + TrickleStream xlsb = new(bytes); + Assert.Throws(() => Excel.FromXlsb(xlsb, leaveOpen)); + Assert.Equal(leaveOpen, xlsb.CanRead); + } + + [Fact] + public void A_Zip_Without_A_Workbook_Closes_The_Stream() + { + TrickleStream stream = new(ZipWithoutWorkbook()); + Assert.Throws(() => Excel.FromXlsx(stream, leaveOpen: false)); + Assert.False(stream.CanRead); + } } } diff --git a/tests/ExcelReader.Tests/Reader/Zip/ZipIndexSourceTests.cs b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexSourceTests.cs index 416ef1cc..1c24085e 100644 --- a/tests/ExcelReader.Tests/Reader/Zip/ZipIndexSourceTests.cs +++ b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexSourceTests.cs @@ -266,11 +266,52 @@ public void A_Failed_Create_Disposes_The_Source() Assert.False(stream.CanRead); } - private static string Outcome(Func open) + private enum Access + { + Part, + Stream, + AsyncPart, + AsyncStream, + } + + private static async Task ReadEntryAsync(ZipIndex index, ZipEntryRef entry, Access access, CancellationToken ct) + { + switch (access) + { + case Access.Part: + using (ZipPart part = index.OpenPart(entry, Counter())) + { + return part.Memory.ToArray(); + } + case Access.AsyncPart: + using (ZipPart part = await index.OpenPartAsync(entry, Counter(), ct)) + { + return part.Memory.ToArray(); + } + case Access.Stream: + using (Stream stream = index.OpenEntryStream(entry, Counter(), ExcelReaderOptions.Default)) + { + using MemoryStream copy = new(); + stream.CopyTo(copy); + return copy.ToArray(); + } + default: + await using (Stream stream = await index.OpenEntryStreamAsync(entry, Counter(), ExcelReaderOptions.Default, ct)) + { + using MemoryStream copy = new(); + await stream.CopyToAsync(copy, ct); + return copy.ToArray(); + } + } + } + + private static async Task OutcomeAsync(Func open, Access access, CancellationToken ct) { try { - using ZipIndex index = ZipIndex.Create(open(), ExcelReaderOptions.Default); + using ZipIndex index = access is Access.AsyncPart or Access.AsyncStream + ? await ZipIndex.CreateAsync(open(), ExcelReaderOptions.Default, ct) + : ZipIndex.Create(open(), ExcelReaderOptions.Default); System.Text.StringBuilder text = new(); foreach (string name in Names) { @@ -281,8 +322,8 @@ private static string Outcome(Func open) } try { - using ZipPart part = index.OpenPart(entry, Counter()); - text.Append(Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(part.Memory.Span))).Append(';'); + byte[] bytes = await ReadEntryAsync(index, entry, access, ct); + text.Append(Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(bytes))).Append(';'); } catch (Exception ex) when (ex is InvalidDataException or NotSupportedException or ExcelLimitExceededException) { @@ -298,8 +339,9 @@ private static string Outcome(Func open) } [Fact] - public void Mutated_Bytes_Give_The_Same_Outcome_From_Memory_And_From_A_Stream() + public async Task Mutated_Bytes_Give_The_Same_Outcome_From_Memory_And_From_A_Stream() { + CancellationToken ct = TestContext.Current.CancellationToken; byte[] pristine = BuildZip(dataDescriptors: false, scale: 1000); uint state = 2024; for (int i = 0; i < 3000; i++) @@ -310,10 +352,58 @@ public void Mutated_Bytes_Give_The_Same_Outcome_From_Memory_And_From_A_Stream() state = (state * 1664525) + 1013904223; mutated[position] = (byte)(state >> 24); - string fromMemory = Outcome(() => ByteSource.FromMemory(mutated)); - string fromStream = Outcome(() => ByteSource.FromStream(new MemoryStream(mutated, writable: false), leaveOpen: false)); - Assert.True(string.Equals(fromMemory, fromStream, StringComparison.Ordinal),$"mutation {i} at byte {position}: memory={fromMemory} stream={fromStream}"); + foreach (Access access in Enum.GetValues()) + { + Access reference = access is Access.Part or Access.AsyncPart ? Access.Part : Access.Stream; + string fromMemory = await OutcomeAsync(() => ByteSource.FromMemory(mutated), reference, ct); + string fromStream = await OutcomeAsync( + () => ByteSource.FromStream(new MemoryStream(mutated, writable: false), leaveOpen: false), access, ct); + Assert.True(string.Equals(fromMemory, fromStream, StringComparison.Ordinal), $"mutation {i} at byte {position}, {access}: memory={fromMemory} stream={fromStream}"); + } + } + } + + private static int CentralRecordOffset(byte[] zipBytes, string entryName) + { + byte[] name = System.Text.Encoding.UTF8.GetBytes(entryName); + for (int i = 0; i + 46 + name.Length <= zipBytes.Length; i++) + { + if (zipBytes.AsSpan(i, 4).SequenceEqual([0x50, 0x4B, 0x01, 0x02]) && zipBytes.AsSpan(i + 46, name.Length).SequenceEqual(name)) + { + return i; + } } + throw new InvalidOperationException("central directory record not found"); + } + + private static byte[] ZipDeclaringTooMuchData() + { + byte[] zipBytes = BuildZip(dataDescriptors: false, scale: 20); + int record = CentralRecordOffset(zipBytes, Names[0]); + System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(zipBytes.AsSpan(record + 24), (uint)(Sizes[0] / 20) + 64); + return zipBytes; + } + + [Theory] + [MemberData(nameof(Kinds))] + public void A_Deflated_Entry_Shorter_Than_Declared_Is_Invalid_Data(Kind kind) + { + using ZipIndex index = ZipIndex.Create(Open(kind, ZipDeclaringTooMuchData()), ExcelReaderOptions.Default); + Assert.True(index.TryGetEntry(System.Text.Encoding.UTF8.GetBytes(Names[0]), out ZipEntryRef entry)); + InvalidDataException ex = Assert.Throws(() => index.OpenPart(entry, Counter()).Dispose()); + Assert.Contains("less data", ex.Message, StringComparison.Ordinal); + } + + [Theory] + [MemberData(nameof(Kinds))] + public async Task A_Deflated_Entry_Shorter_Than_Declared_Is_Invalid_Data_Asynchronously(Kind kind) + { + CancellationToken ct = TestContext.Current.CancellationToken; + using ZipIndex index = await ZipIndex.CreateAsync(Open(kind, ZipDeclaringTooMuchData()), ExcelReaderOptions.Default, ct); + Assert.True(index.TryGetEntry(System.Text.Encoding.UTF8.GetBytes(Names[0]), out ZipEntryRef entry)); + InvalidDataException ex = await Assert.ThrowsAsync( + async () => (await index.OpenPartAsync(entry, Counter(), ct)).Dispose()); + Assert.Contains("less data", ex.Message, StringComparison.Ordinal); } [Fact] diff --git a/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs index 8903de08..f764304e 100644 --- a/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs +++ b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs @@ -4,6 +4,7 @@ using System.Runtime.InteropServices; using System.Text; using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Sources; using ExcelReader.Core.Reader.Zip; namespace ExcelReader.Tests.Reader.Zip @@ -505,6 +506,258 @@ public void LocalExtraLengthPushingDataPastTheFileEndThrowsInvalidDataOnOpen() Assert.Contains("past the end", ex.Message, StringComparison.OrdinalIgnoreCase); } + private static readonly bool[] Modes = [false, true]; + + private static async Task CreatePositionalAsync(byte[] bytes, bool useAsync, CancellationToken ct) + { + ByteSource source = ByteSource.FromStream(new MemoryStream(bytes, writable: false), leaveOpen: false); + return useAsync + ? await ZipIndex.CreateAsync(source, ExcelReaderOptions.Default, ct) + : ZipIndex.Create(source, ExcelReaderOptions.Default); + } + + private static async Task CreateFailsAsync(byte[] bytes, bool useAsync, CancellationToken ct) + { + return await Assert.ThrowsAsync(async () => (await CreatePositionalAsync(bytes, useAsync, ct)).Dispose()); + } + + private static async Task OpenPartPositionalAsync(ZipIndex index, ZipEntryRef entry, bool useAsync, CancellationToken ct) + { + DecompressedByteCounter counter = new(0); + return useAsync ? await index.OpenPartAsync(entry, counter, ct) : index.OpenPart(entry, counter); + } + + private static async Task OpenPartFailsAsync(ZipIndex index, ZipEntryRef entry, bool useAsync, CancellationToken ct) + { + return await Assert.ThrowsAsync(async () => (await OpenPartPositionalAsync(index, entry, useAsync, ct)).Dispose()); + } + + [Fact] + public async Task PositionalValidZip64ArchiveOpensTheStoredEntry() + { + CancellationToken ct = TestContext.Current.CancellationToken; + foreach (bool useAsync in Modes) + { + Zip64Layout zip = BuildStoredZip64(Zip64Payload); + using ZipIndex index = await CreatePositionalAsync(zip.Bytes, useAsync, ct); + Assert.False(index.HasMemory); + Assert.True(index.TryGetEntry("a.txt"u8, out ZipEntryRef entry)); + Assert.Equal(0, entry.LocalHeaderOffset); + Assert.Equal(Zip64Payload.Length, entry.CompressedSize); + Assert.Equal(Zip64Payload.Length, entry.UncompressedSize); + using ZipPart part = await OpenPartPositionalAsync(index, entry, useAsync, ct); + Assert.Equal(Zip64Payload, part.Memory.ToArray()); + } + } + + [Fact] + public async Task PositionalHugeZip64EocdLocatorOffsetThrowsInsteadOfWrapping() + { + CancellationToken ct = TestContext.Current.CancellationToken; + foreach (bool useAsync in Modes) + { + byte[] bytes = new byte[42]; + WriteZip64Locator(bytes, 0, zip64EocdOffset: long.MaxValue - 2); + WriteEocd(bytes, 20, declaredCount: 0xFFFF, cdSize: Zip64SentinelU32, cdOffset: Zip64SentinelU32); + + InvalidDataException ex = await CreateFailsAsync(bytes, useAsync, ct); + Assert.Contains("ZIP64", ex.Message, StringComparison.Ordinal); + } + } + + [Fact] + public async Task PositionalHugeZip64CentralDirectorySizeThrowsInsteadOfWrapping() + { + CancellationToken ct = TestContext.Current.CancellationToken; + foreach (bool useAsync in Modes) + { + byte[] bytes = new byte[98]; + BinaryPrimitives.WriteInt32LittleEndian(bytes.AsSpan(0), 0x06064b50); + BinaryPrimitives.WriteInt64LittleEndian(bytes.AsSpan(32), 1); + BinaryPrimitives.WriteInt64LittleEndian(bytes.AsSpan(40), long.MaxValue - 2); + BinaryPrimitives.WriteInt64LittleEndian(bytes.AsSpan(48), 5); + WriteZip64Locator(bytes, 56, zip64EocdOffset: 0); + WriteEocd(bytes, 76, declaredCount: 0xFFFF, cdSize: Zip64SentinelU32, cdOffset: Zip64SentinelU32); + + InvalidDataException ex = await CreateFailsAsync(bytes, useAsync, ct); + Assert.Contains("central directory", ex.Message, StringComparison.OrdinalIgnoreCase); + } + } + + [Fact] + public async Task PositionalNegativeZip64SizesThrowInvalidDataException() + { + CancellationToken ct = TestContext.Current.CancellationToken; + foreach (bool useAsync in Modes) + { + byte[] uncompressed = BuildCdWithZip64Field(compressedSize: 5, uncompressedSize: Zip64SentinelU32, zip64Value: -1); + Assert.Contains("negative", (await CreateFailsAsync(uncompressed, useAsync, ct)).Message, StringComparison.OrdinalIgnoreCase); + + byte[] compressed = BuildCdWithZip64Field(compressedSize: Zip64SentinelU32, uncompressedSize: 5, zip64Value: -1); + Assert.Contains("negative", (await CreateFailsAsync(compressed, useAsync, ct)).Message, StringComparison.OrdinalIgnoreCase); + } + } + + [Theory] + [InlineData(long.MaxValue)] + [InlineData(long.MaxValue - 29)] + [InlineData(long.MaxValue - 30)] + [InlineData((long)int.MaxValue + 1)] + [InlineData(int.MaxValue)] + public async Task PositionalHugeZip64LocalHeaderOffsetThrowsInvalidDataOnOpen(long localOffset) + { + CancellationToken ct = TestContext.Current.CancellationToken; + foreach (bool useAsync in Modes) + { + Zip64Layout zip = BuildStoredZip64(Zip64Payload, localOffset: localOffset); + + using ZipIndex index = await CreatePositionalAsync(zip.Bytes, useAsync, ct); + Assert.True(index.TryGetEntry("a.txt"u8, out ZipEntryRef entry)); + Assert.Equal(localOffset, entry.LocalHeaderOffset); + InvalidDataException ex = await OpenPartFailsAsync(index, entry, useAsync, ct); + Assert.Contains("local file header", ex.Message, StringComparison.OrdinalIgnoreCase); + } + } + + [Fact] + public async Task PositionalLocalHeaderOffsetJustInsideTheFileEndThrowsInvalidDataOnOpen() + { + CancellationToken ct = TestContext.Current.CancellationToken; + foreach (bool useAsync in Modes) + { + Zip64Layout probe = BuildStoredZip64(Zip64Payload); + Zip64Layout zip = BuildStoredZip64(Zip64Payload, localOffset: probe.Bytes.Length - 29); + + using ZipIndex index = await CreatePositionalAsync(zip.Bytes, useAsync, ct); + Assert.True(index.TryGetEntry("a.txt"u8, out ZipEntryRef entry)); + InvalidDataException ex = await OpenPartFailsAsync(index, entry, useAsync, ct); + Assert.Contains("out of range", ex.Message, StringComparison.OrdinalIgnoreCase); + } + } + + [Theory] + [InlineData(long.MaxValue)] + [InlineData(long.MaxValue - 50)] + [InlineData((long)int.MaxValue + 1)] + [InlineData(int.MaxValue)] + public async Task PositionalHugeZip64CompressedSizeThrowsInvalidDataOnOpen(long compressedSize) + { + CancellationToken ct = TestContext.Current.CancellationToken; + foreach (bool useAsync in Modes) + { + Zip64Layout zip = BuildStoredZip64(Zip64Payload, compressedSize: compressedSize); + + using ZipIndex index = await CreatePositionalAsync(zip.Bytes, useAsync, ct); + Assert.True(index.TryGetEntry("a.txt"u8, out ZipEntryRef entry)); + Assert.Equal(compressedSize, entry.CompressedSize); + InvalidDataException ex = await OpenPartFailsAsync(index, entry, useAsync, ct); + Assert.Contains("past the end", ex.Message, StringComparison.OrdinalIgnoreCase); + if (useAsync) + { + await Assert.ThrowsAsync( + async () => (await index.OpenEntryStreamAsync(entry, new DecompressedByteCounter(0), ExcelReaderOptions.Default, ct)).Dispose()); + } + else + { + Assert.Throws( + () => index.OpenEntryStream(entry, new DecompressedByteCounter(0), ExcelReaderOptions.Default)); + } + } + } + + [Fact] + public async Task PositionalCompressedSizeEndingExactlyAtTheFileEndIsAcceptedAndOneMoreIsRejected() + { + CancellationToken ct = TestContext.Current.CancellationToken; + foreach (bool useAsync in Modes) + { + Zip64Layout probe = BuildStoredZip64(Zip64Payload); + long exact = probe.Bytes.Length - probe.DataOffset; + + Zip64Layout fits = BuildStoredZip64(Zip64Payload, compressedSize: exact, uncompressedSize: exact); + using (ZipIndex index = await CreatePositionalAsync(fits.Bytes, useAsync, ct)) + { + Assert.True(index.TryGetEntry("a.txt"u8, out ZipEntryRef entry)); + using ZipPart part = await OpenPartPositionalAsync(index, entry, useAsync, ct); + Assert.Equal(exact, part.Memory.Length); + } + + Zip64Layout overruns = BuildStoredZip64(Zip64Payload, compressedSize: exact + 1); + using (ZipIndex index = await CreatePositionalAsync(overruns.Bytes, useAsync, ct)) + { + Assert.True(index.TryGetEntry("a.txt"u8, out ZipEntryRef entry)); + await OpenPartFailsAsync(index, entry, useAsync, ct); + } + } + } + + [Theory] + [InlineData(0x8000000000000000UL)] + [InlineData(ulong.MaxValue)] + public async Task PositionalZip64LocalHeaderOffsetAboveLongMaxIsRejectedAsNegative(ulong localOffset) + { + CancellationToken ct = TestContext.Current.CancellationToken; + foreach (bool useAsync in Modes) + { + Zip64Layout zip = BuildStoredZip64(Zip64Payload, localOffset: unchecked((long)localOffset)); + + InvalidDataException ex = await CreateFailsAsync(zip.Bytes, useAsync, ct); + Assert.Contains("negative", ex.Message, StringComparison.OrdinalIgnoreCase); + } + } + + [Theory] + [InlineData(0x8000000000000000UL)] + [InlineData(ulong.MaxValue)] + public async Task PositionalZip64LocatorOffsetAboveLongMaxIsRejectedNotIgnored(ulong zip64EocdOffset) + { + CancellationToken ct = TestContext.Current.CancellationToken; + foreach (bool useAsync in Modes) + { + Zip64Layout zip = BuildStoredZip64(Zip64Payload); + BinaryPrimitives.WriteUInt64LittleEndian(zip.Bytes.AsSpan(zip.LocatorOffset + 8), zip64EocdOffset); + + InvalidDataException ex = await CreateFailsAsync(zip.Bytes, useAsync, ct); + Assert.Contains("ZIP64", ex.Message, StringComparison.Ordinal); + } + } + + [Theory] + [InlineData(long.MaxValue, 1UL)] + [InlineData(long.MaxValue, (ulong)long.MaxValue)] + [InlineData(1L, (ulong)long.MaxValue)] + [InlineData(0L, ulong.MaxValue)] + [InlineData(-1L, 0UL)] + [InlineData(long.MinValue, 100UL)] + [InlineData(0L, 0x8000000000000000UL)] + public async Task PositionalZip64CentralDirectoryBoundsThatWouldOverflowAreRejected(long cdOffset, ulong cdSize) + { + CancellationToken ct = TestContext.Current.CancellationToken; + foreach (bool useAsync in Modes) + { + Zip64Layout zip = BuildStoredZip64(Zip64Payload); + BinaryPrimitives.WriteUInt64LittleEndian(zip.Bytes.AsSpan(zip.Zip64EocdOffset + 40), cdSize); + BinaryPrimitives.WriteInt64LittleEndian(zip.Bytes.AsSpan(zip.Zip64EocdOffset + 48), cdOffset); + + InvalidDataException ex = await CreateFailsAsync(zip.Bytes, useAsync, ct); + Assert.Contains("central directory", ex.Message, StringComparison.OrdinalIgnoreCase); + } + } + + [Fact] + public async Task PositionalZip64CentralDirectorySizeOneByteLongerThanItsRecordsIsRejected() + { + CancellationToken ct = TestContext.Current.CancellationToken; + foreach (bool useAsync in Modes) + { + Zip64Layout zip = BuildStoredZip64(Zip64Payload); + long cdSize = BinaryPrimitives.ReadInt64LittleEndian(zip.Bytes.AsSpan(zip.Zip64EocdOffset + 40)); + BinaryPrimitives.WriteInt64LittleEndian(zip.Bytes.AsSpan(zip.Zip64EocdOffset + 40), cdSize + 1); + + await CreateFailsAsync(zip.Bytes, useAsync, ct); + } + } + private readonly record struct Zip64Layout(byte[] Bytes, int DataOffset, int Zip64EocdOffset, int LocatorOffset); private static Zip64Layout BuildStoredZip64(byte[] payload, long? localOffset = null, long? compressedSize = null, long? uncompressedSize = null) From fd30186aa7543a11cbd0606f2c603a0078d3a975 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Tue, 6 Oct 2026 03:09:37 -0300 Subject: [PATCH 13/28] fix(core): restore stream position on failed detection and wrap only exact stream types Co-Authored-By: Claude Sonnet 5.5 --- src/ExcelReader.Core/Reader/Excel.cs | 27 +++++++++++++--- .../Reader/Sources/ByteSource.cs | 6 ++-- .../Reader/SourceParityTests.cs | 18 +++++++++++ .../Reader/Sources/ByteSourceTests.cs | 31 +++++++++++++++++++ .../Reader/Zip/ZipIndexTests.cs | 2 +- 5 files changed, 76 insertions(+), 8 deletions(-) diff --git a/src/ExcelReader.Core/Reader/Excel.cs b/src/ExcelReader.Core/Reader/Excel.cs index b668631f..001b3f9b 100644 --- a/src/ExcelReader.Core/Reader/Excel.cs +++ b/src/ExcelReader.Core/Reader/Excel.cs @@ -803,8 +803,17 @@ private static ExcelFileFormat DetectSeekable(Stream stream, bool leaveOpen, Exc ? ExcelFileFormat.EncryptedOoxml : ExcelFileFormat.Xls; } - zip = ZipIndex.Create(ByteSource.FromStream(stream, leaveOpen), options); - stream.Position = start; + try + { + zip = ZipIndex.Create(ByteSource.FromStream(stream, leaveOpen), options); + } + finally + { + if (stream.CanSeek) + { + stream.Position = start; + } + } return ClassifyZip(zip); } @@ -839,8 +848,18 @@ private static ExcelFileFormat DetectSeekable(Stream stream, bool leaveOpen, Exc return (cfbFormat, null); } ByteSource source = await ByteSource.FromStreamAsync(stream, leaveOpen, ct).ConfigureAwait(false); - ZipIndex zip = await ZipIndex.CreateAsync(source, options, ct).ConfigureAwait(false); - stream.Position = start; + ZipIndex zip; + try + { + zip = await ZipIndex.CreateAsync(source, options, ct).ConfigureAwait(false); + } + finally + { + if (stream.CanSeek) + { + stream.Position = start; + } + } return (ClassifyZip(zip), zip); } diff --git a/src/ExcelReader.Core/Reader/Sources/ByteSource.cs b/src/ExcelReader.Core/Reader/Sources/ByteSource.cs index 7005cd6b..1801ead6 100644 --- a/src/ExcelReader.Core/Reader/Sources/ByteSource.cs +++ b/src/ExcelReader.Core/Reader/Sources/ByteSource.cs @@ -115,14 +115,14 @@ internal static async ValueTask FromStreamAsync(Stream stream, bool private static bool TryWrapSeekable(Stream stream, bool leaveOpen, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out ByteSource? source) { IDisposable? owner = leaveOpen ? null : stream; - if (stream is MemoryStream memory && memory.TryGetBuffer(out ArraySegment segment)) + if (stream.GetType() == typeof(MemoryStream) && ((MemoryStream)stream).TryGetBuffer(out ArraySegment segment)) { source = new MemoryByteSource(segment, owner); return true; } - if (stream is FileStream file && file.CanSeek) + if (stream.GetType() == typeof(FileStream) && stream.CanSeek) { - source = new FileByteSource(file.SafeFileHandle, file.Length, owner); + source = new FileByteSource(((FileStream)stream).SafeFileHandle, stream.Length, owner); return true; } if (stream.CanSeek) diff --git a/tests/ExcelReader.Tests/Reader/SourceParityTests.cs b/tests/ExcelReader.Tests/Reader/SourceParityTests.cs index b1dea41c..db336e24 100644 --- a/tests/ExcelReader.Tests/Reader/SourceParityTests.cs +++ b/tests/ExcelReader.Tests/Reader/SourceParityTests.cs @@ -515,5 +515,23 @@ public void A_Zip_Without_A_Workbook_Closes_The_Stream() Assert.Throws(() => Excel.FromXlsx(stream, leaveOpen: false)); Assert.False(stream.CanRead); } + + [Fact] + public void A_Failed_Detection_Restores_The_Stream_Position() + { + TrickleStream stream = new(CorruptZip()); + Assert.Throws(() => Excel.DetectFileFormat(stream)); + Assert.Equal(0, stream.Position); + Assert.True(stream.CanRead); + } + + [Fact] + public async Task A_Failed_Asynchronous_Detection_Restores_The_Stream_Position() + { + TrickleStream stream = new(CorruptZip()); + await Assert.ThrowsAsync(async () => await Excel.DetectFileFormatAsync(stream, TestContext.Current.CancellationToken)); + Assert.Equal(0, stream.Position); + Assert.True(stream.CanRead); + } } } diff --git a/tests/ExcelReader.Tests/Reader/Sources/ByteSourceTests.cs b/tests/ExcelReader.Tests/Reader/Sources/ByteSourceTests.cs index 90c37ad5..8c684040 100644 --- a/tests/ExcelReader.Tests/Reader/Sources/ByteSourceTests.cs +++ b/tests/ExcelReader.Tests/Reader/Sources/ByteSourceTests.cs @@ -188,5 +188,36 @@ public async Task FromStreamAsync_Buffers_A_NonSeekable_Stream() Assert.True(source.TryGetMemory(out ReadOnlyMemory memory)); Assert.True(memory.Span.SequenceEqual(Payload)); } + + private sealed class InvertingMemoryStream(byte[] bytes) : MemoryStream(bytes, 0, bytes.Length, writable: false, publiclyVisible: true) + { + public override int Read(byte[] buffer, int offset, int count) + { + return Read(buffer.AsSpan(offset, count)); + } + + public override int Read(Span buffer) + { + int read = base.Read(buffer); + for (int i = 0; i < read; i++) + { + buffer[i] ^= 0xFF; + } + return read; + } + } + + [Fact] + public void A_MemoryStream_Subclass_Is_Read_Through_Its_Own_Read() + { + using ByteSource source = ByteSource.FromStream(new InvertingMemoryStream(Payload), leaveOpen: false); + Assert.False(source.TryGetMemory(out _)); + byte[] read = new byte[64]; + source.ReadExactly(100, read); + for (int i = 0; i < read.Length; i++) + { + Assert.Equal((byte)(Payload[100 + i] ^ 0xFF), read[i]); + } + } } } diff --git a/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs index f764304e..8a765067 100644 --- a/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs +++ b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs @@ -166,7 +166,7 @@ public void CreateThrowsWhenNoEndOfCentralDirectoryRecordExists() } [Fact] - public void MutatedZipBytesNeverCrashTheMemoryIndex() + public void MutatedZipBytesNeverCrashTheIndex() { using MemoryStream built = WorkbookBuilder.BuildMultiSheet( sheets: From d408b0f6a6495ef08e31d43935af3a56333f45a3 Mon Sep 17 00:00:00 2001 From: GabrielMarquezMatte Date: Tue, 6 Oct 2026 03:16:51 -0300 Subject: [PATCH 14/28] test(core): make the inverting MemoryStream subclass not recurse Co-Authored-By: Claude Sonnet 5.5 --- .../Reader/Sources/ByteSourceTests.cs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/tests/ExcelReader.Tests/Reader/Sources/ByteSourceTests.cs b/tests/ExcelReader.Tests/Reader/Sources/ByteSourceTests.cs index 8c684040..b60e88f7 100644 --- a/tests/ExcelReader.Tests/Reader/Sources/ByteSourceTests.cs +++ b/tests/ExcelReader.Tests/Reader/Sources/ByteSourceTests.cs @@ -193,16 +193,19 @@ private sealed class InvertingMemoryStream(byte[] bytes) : MemoryStream(bytes, 0 { public override int Read(byte[] buffer, int offset, int count) { - return Read(buffer.AsSpan(offset, count)); + int read = base.Read(buffer, offset, count); + for (int i = 0; i < read; i++) + { + buffer[offset + i] ^= 0xFF; + } + return read; } public override int Read(Span buffer) { - int read = base.Read(buffer); - for (int i = 0; i < read; i++) - { - buffer[i] ^= 0xFF; - } + byte[] scratch = new byte[buffer.Length]; + int read = Read(scratch, 0, scratch.Length); + scratch.AsSpan(0, read).CopyTo(buffer); return read; } } From 08933cea3398bc8ce7b1f626dab7b64dc9dcb6fb Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Tue, 6 Oct 2026 10:26:47 -0300 Subject: [PATCH 15/28] fix(core): count decompressed bytes atomically MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DecompressedByteCounter.Add and Remaining are now thread-safe via Interlocked.Add and Interlocked.Read, enabling concurrent sheet inflation under a shared decompression limit. Once any thread exceeds the limit, all subsequent Add calls throw—captured atomically in _total regardless of order. Co-Authored-By: Claude Haiku 4.5 --- .../Reader/Zip/LimitedReadStream.cs | 9 ++-- .../Internal/DecompressedByteCounterTests.cs | 44 +++++++++++++++++++ 2 files changed, 48 insertions(+), 5 deletions(-) create mode 100644 tests/ExcelReader.Tests/Reader/Internal/DecompressedByteCounterTests.cs diff --git a/src/ExcelReader.Core/Reader/Zip/LimitedReadStream.cs b/src/ExcelReader.Core/Reader/Zip/LimitedReadStream.cs index 8bfb9641..185a25ba 100644 --- a/src/ExcelReader.Core/Reader/Zip/LimitedReadStream.cs +++ b/src/ExcelReader.Core/Reader/Zip/LimitedReadStream.cs @@ -18,15 +18,14 @@ internal void Add(long bytes) { return; } - long total = checked(_total + bytes); - if (total > _limit) + long total = Interlocked.Add(ref _total, bytes); + if (total > _limit || total < 0) { - throw new ExcelLimitExceededException(_limitName, _limit, total); + throw new ExcelLimitExceededException(_limitName, _limit, total < 0 ? long.MaxValue : total); } - _total = total; } - internal long Remaining => _limit <= 0 ? long.MaxValue : _limit - _total; + internal long Remaining => _limit <= 0 ? long.MaxValue : _limit - Interlocked.Read(ref _total); } internal sealed class LimitedReadStream : Stream diff --git a/tests/ExcelReader.Tests/Reader/Internal/DecompressedByteCounterTests.cs b/tests/ExcelReader.Tests/Reader/Internal/DecompressedByteCounterTests.cs new file mode 100644 index 00000000..82252468 --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Internal/DecompressedByteCounterTests.cs @@ -0,0 +1,44 @@ +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Zip; + +namespace ExcelReader.Tests.Reader.Internal +{ + public sealed class DecompressedByteCounterTests + { + [Fact] + public void Concurrent_Adds_Are_All_Counted() + { + const int Threads = 16; + const int PerThread = 20_000; + DecompressedByteCounter counter = new(Threads * PerThread); + + Parallel.For(0, Threads, new ParallelOptions { MaxDegreeOfParallelism = Threads }, _ => + { + for (int i = 0; i < PerThread; i++) + { + counter.Add(1); + } + }); + + Assert.Equal(0, counter.Remaining); + Assert.Throws(() => counter.Add(1)); + } + + [Fact] + public void Passing_The_Limit_On_One_Thread_Fails_The_Others_Too() + { + DecompressedByteCounter counter = new(100); + Assert.Throws(() => counter.Add(101)); + Assert.Throws(() => counter.Add(1)); + } + + [Fact] + public void A_Counter_Without_A_Limit_Never_Throws() + { + DecompressedByteCounter counter = new(0); + counter.Add(long.MaxValue); + counter.Add(long.MaxValue); + Assert.Equal(long.MaxValue, counter.Remaining); + } + } +} From 0e99b30b67d0c0b5954625211fbe9e57d6717da3 Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Tue, 6 Oct 2026 10:33:09 -0300 Subject: [PATCH 16/28] refactor(core): publish cached shared strings with compare-and-swap Replace lazy-initialization with thread-safe compare-and-swap in Cell.GetString() and each reader's SharedStringCache property. Materializes cache array once per reader; filled slots never change, ensuring enumerators on different threads see consistent string references and cached values. Co-Authored-By: Claude Haiku 4.5 --- src/ExcelReader.Core/Reader/Cell.cs | 10 ++- src/ExcelReader.Core/Reader/Xls/XlsReader.cs | 8 ++- .../Reader/Xlsb/XlsbReader.cs | 8 ++- .../Reader/Xlsx/XlsxReader.cs | 8 ++- .../Reader/Internal/SharedStringCacheTests.cs | 62 +++++++++++++++++++ 5 files changed, 91 insertions(+), 5 deletions(-) create mode 100644 tests/ExcelReader.Tests/Reader/Internal/SharedStringCacheTests.cs diff --git a/src/ExcelReader.Core/Reader/Cell.cs b/src/ExcelReader.Core/Reader/Cell.cs index c12e2d14..27ec5356 100644 --- a/src/ExcelReader.Core/Reader/Cell.cs +++ b/src/ExcelReader.Core/Reader/Cell.cs @@ -354,8 +354,14 @@ public string GetString() } if (_sharedCache is not null && !Value.IsEmpty && (uint)_sharedIndex < (uint)_sharedCache.Length) { - ref string? cached = ref _sharedCache[_sharedIndex]; - return cached ??= Encoding.UTF8.GetString(Value); + ref string? slot = ref _sharedCache[_sharedIndex]; + string? cached = Volatile.Read(ref slot); + if (cached is not null) + { + return cached; + } + string created = Encoding.UTF8.GetString(Value); + return Interlocked.CompareExchange(ref slot, created, null) ?? created; } if (_contentCache is not null && !Value.IsEmpty) { diff --git a/src/ExcelReader.Core/Reader/Xls/XlsReader.cs b/src/ExcelReader.Core/Reader/Xls/XlsReader.cs index 2d09e80d..15ed04fe 100644 --- a/src/ExcelReader.Core/Reader/Xls/XlsReader.cs +++ b/src/ExcelReader.Core/Reader/Xls/XlsReader.cs @@ -84,7 +84,13 @@ public ExcelSheetVisibility SheetVisibilityAt(int index) internal ReadOnlySpan SharedSpan => _sharedFlat; - internal string?[] SharedStringCache => _sharedStringCache ??= WorkbookLookups.CreateSharedStringCache(_sharedOffsets); + internal string?[] SharedStringCache => Volatile.Read(ref _sharedStringCache) ?? CreateSharedStringCache(); + + private string?[] CreateSharedStringCache() + { + string?[] created = WorkbookLookups.CreateSharedStringCache(_sharedOffsets); + return Interlocked.CompareExchange(ref _sharedStringCache, created, null) ?? created; + } internal bool IsDateStyle(int style) { diff --git a/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.cs b/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.cs index a07bd97e..c84b0ed0 100644 --- a/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.cs +++ b/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.cs @@ -207,7 +207,13 @@ public void MoveToSheet(int index) internal ReadOnlySpan SharedSpan => _sharedFlat; - internal string?[] SharedStringCache => _sharedStringCache ??= WorkbookLookups.CreateSharedStringCache(_sharedOffsets); + internal string?[] SharedStringCache => Volatile.Read(ref _sharedStringCache) ?? CreateSharedStringCache(); + + private string?[] CreateSharedStringCache() + { + string?[] created = WorkbookLookups.CreateSharedStringCache(_sharedOffsets); + return Interlocked.CompareExchange(ref _sharedStringCache, created, null) ?? created; + } internal bool IsDateStyle(int style) { diff --git a/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.cs b/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.cs index 316601a4..0c124f96 100644 --- a/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.cs +++ b/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.cs @@ -179,7 +179,13 @@ IExcelRowEnumerator IExcelRowReader.GetAsyncEnumerator(Canc internal ReadOnlySpan SharedSpan => _sharedFlat; - internal string?[] SharedStringCache => _sharedStringCache ??= WorkbookLookups.CreateSharedStringCache(_sharedOffsets); + internal string?[] SharedStringCache => Volatile.Read(ref _sharedStringCache) ?? CreateSharedStringCache(); + + private string?[] CreateSharedStringCache() + { + string?[] created = WorkbookLookups.CreateSharedStringCache(_sharedOffsets); + return Interlocked.CompareExchange(ref _sharedStringCache, created, null) ?? created; + } /// public void Dispose() diff --git a/tests/ExcelReader.Tests/Reader/Internal/SharedStringCacheTests.cs b/tests/ExcelReader.Tests/Reader/Internal/SharedStringCacheTests.cs new file mode 100644 index 00000000..9c3fe56e --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Internal/SharedStringCacheTests.cs @@ -0,0 +1,62 @@ +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Xlsx; +using ExcelReader.Core.Writer.Xlsx; + +namespace ExcelReader.Tests.Reader.Internal +{ + public sealed class SharedStringCacheTests + { + private static byte[] BuildXlsx() + { + using MemoryStream buffer = new(); + using (XlsxWorkbookWriter workbook = XlsxWorkbookWriter.Create(buffer, leaveOpen: true, new XlsxWriterOptions { UseSharedStrings = true })) + { + XlsxSheetWriter sheet = workbook.AddSheet("s"); + for (int r = 0; r < 50; r++) + { + using XlsxRowWriter row = sheet.StartRow(); + row.Write("value-" + (r % 5).ToString(System.Globalization.CultureInfo.InvariantCulture)); + } + sheet.End(); + sheet.Dispose(); + workbook.End(); + } + return buffer.ToArray(); + } + + private static List FirstColumn(XlsxReader reader) + { + List values = []; + using XlsxReader.Enumerator e = reader.GetEnumerator(); + while (e.MoveNext()) + { + values.Add(e.Current[0].GetString()); + } + return values; + } + + [Fact] + public void A_Shared_String_Is_Materialized_Once_Per_Reader() + { + using XlsxReader reader = Excel.FromXlsx(BuildXlsx()); + List first = FirstColumn(reader); + List second = FirstColumn(reader); + + Assert.Equal(50, first.Count); + for (int i = 0; i < first.Count; i++) + { + Assert.Same(first[i], second[i]); + Assert.Same(first[i % 5], first[i]); + } + } + + [Fact] + public void The_Cache_Array_Is_One_Instance() + { + using XlsxReader reader = Excel.FromXlsx(BuildXlsx()); + _ = FirstColumn(reader); + string?[] seen = reader.SharedStringCache; + Parallel.For(0, 32, _ => Assert.Same(seen, reader.SharedStringCache)); + } + } +} From e686e737a5e21b748a7778eb6b8a21806f8b023a Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Tue, 6 Oct 2026 10:38:26 -0300 Subject: [PATCH 17/28] feat(core): add OnceGate and ReaderLifetime for shared reader state OnceGate runs a load once for multiple callers. A failure is recorded and rethrown without reloading; cancellation runs reset and lets the next caller load fresh. Sync and async paths share one load. ReaderLifetime is a reference-counted lifetime tracker. The reader holds one reference, each enumerator another. Release runs when the last reference goes. Close is idempotent and prevents new acquisitions once called. Co-Authored-By: Claude Haiku 4.5 --- .../Reader/Internal/OnceGate.cs | 95 +++++++++++++++ .../Reader/Internal/ReaderLifetime.cs | 41 +++++++ .../Reader/Internal/OnceGateTests.cs | 111 ++++++++++++++++++ .../Reader/Internal/ReaderLifetimeTests.cs | 67 +++++++++++ 4 files changed, 314 insertions(+) create mode 100644 src/ExcelReader.Core/Reader/Internal/OnceGate.cs create mode 100644 src/ExcelReader.Core/Reader/Internal/ReaderLifetime.cs create mode 100644 tests/ExcelReader.Tests/Reader/Internal/OnceGateTests.cs create mode 100644 tests/ExcelReader.Tests/Reader/Internal/ReaderLifetimeTests.cs diff --git a/src/ExcelReader.Core/Reader/Internal/OnceGate.cs b/src/ExcelReader.Core/Reader/Internal/OnceGate.cs new file mode 100644 index 00000000..88a18938 --- /dev/null +++ b/src/ExcelReader.Core/Reader/Internal/OnceGate.cs @@ -0,0 +1,95 @@ +using System.Runtime.ExceptionServices; + +namespace ExcelReader.Core.Reader.Internal +{ + /// + /// Runs a load once for however many callers ask. A failure is kept and rethrown; a cancelled + /// load is undone so the next caller starts clean. + /// + internal sealed class OnceGate : IDisposable + { + private readonly SemaphoreSlim _gate = new(1, 1); + private volatile bool _done; + private ExceptionDispatchInfo? _failure; + + internal bool IsDone => _done; + + internal void Run(TState state, Action load, Action reset) + { + if (_done) + { + return; + } + _gate.Wait(); + try + { + if (_done) + { + return; + } + _failure?.Throw(); + try + { + load(state); + } + catch (OperationCanceledException) + { + reset(state); + throw; + } + catch (Exception ex) + { + _failure = ExceptionDispatchInfo.Capture(ex); + throw; + } + _done = true; + } + finally + { + _gate.Release(); + } + } + + internal async ValueTask RunAsync( + TState state, Func load, Action reset, CancellationToken ct) + { + if (_done) + { + return; + } + await _gate.WaitAsync(ct).ConfigureAwait(false); + try + { + if (_done) + { + return; + } + _failure?.Throw(); + try + { + await load(state, ct).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + reset(state); + throw; + } + catch (Exception ex) + { + _failure = ExceptionDispatchInfo.Capture(ex); + throw; + } + _done = true; + } + finally + { + _gate.Release(); + } + } + + public void Dispose() + { + _gate.Dispose(); + } + } +} diff --git a/src/ExcelReader.Core/Reader/Internal/ReaderLifetime.cs b/src/ExcelReader.Core/Reader/Internal/ReaderLifetime.cs new file mode 100644 index 00000000..4ea572dd --- /dev/null +++ b/src/ExcelReader.Core/Reader/Internal/ReaderLifetime.cs @@ -0,0 +1,41 @@ +namespace ExcelReader.Core.Reader.Internal +{ + /// + /// Keeps a reader's resources alive while any enumerator still uses them: the reader holds one + /// reference, each enumerator another, and the release runs when the last one goes. + /// + internal sealed class ReaderLifetime(Action release) + { + private int _count = 1; + private int _closed; + + internal void Acquire(object owner) + { + while (true) + { + int current = Volatile.Read(ref _count); + ObjectDisposedException.ThrowIf(current == 0 || Volatile.Read(ref _closed) != 0, owner); + if (Interlocked.CompareExchange(ref _count, current + 1, current) == current) + { + return; + } + } + } + + internal void Release() + { + if (Interlocked.Decrement(ref _count) == 0) + { + release(); + } + } + + internal void Close() + { + if (Interlocked.Exchange(ref _closed, 1) == 0) + { + Release(); + } + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Internal/OnceGateTests.cs b/tests/ExcelReader.Tests/Reader/Internal/OnceGateTests.cs new file mode 100644 index 00000000..de50b555 --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Internal/OnceGateTests.cs @@ -0,0 +1,111 @@ +using ExcelReader.Core.Reader.Internal; + +namespace ExcelReader.Tests.Reader.Internal +{ + public sealed class OnceGateTests + { + private sealed class Probe + { + internal int Loads; + internal int Resets; + internal Exception? Fail; + } + + private static void Load(Probe probe) + { + Interlocked.Increment(ref probe.Loads); + if (probe.Fail is not null) + { + throw probe.Fail; + } + } + + private static void Reset(Probe probe) + { + Interlocked.Increment(ref probe.Resets); + } + + [Fact] + public void Load_Runs_Once_For_Many_Concurrent_Callers() + { + using OnceGate gate = new(); + Probe probe = new(); + Parallel.For(0, 64, _ => gate.Run(probe, Load, Reset)); + Assert.Equal(1, probe.Loads); + Assert.True(gate.IsDone); + } + + [Fact] + public void A_Failure_Is_Rethrown_Without_Loading_Again() + { + using OnceGate gate = new(); + Probe probe = new() { Fail = new InvalidDataException("bad table") }; + + InvalidDataException first = Assert.Throws(() => gate.Run(probe, Load, Reset)); + probe.Fail = null; + InvalidDataException second = Assert.Throws(() => gate.Run(probe, Load, Reset)); + + Assert.Same(first, second); + Assert.Equal(1, probe.Loads); + Assert.Equal(0, probe.Resets); + Assert.False(gate.IsDone); + } + + [Fact] + public void Cancellation_Resets_And_Lets_The_Next_Caller_Load() + { + using OnceGate gate = new(); + Probe probe = new() { Fail = new OperationCanceledException() }; + + Assert.Throws(() => gate.Run(probe, Load, Reset)); + Assert.Equal(1, probe.Resets); + Assert.False(gate.IsDone); + + probe.Fail = null; + gate.Run(probe, Load, Reset); + Assert.Equal(2, probe.Loads); + Assert.True(gate.IsDone); + } + + [Fact] + public async Task RunAsync_Follows_The_Same_Rules() + { + CancellationToken ct = TestContext.Current.CancellationToken; + using OnceGate gate = new(); + Probe probe = new() { Fail = new OperationCanceledException() }; + + static ValueTask LoadAsync(Probe p, CancellationToken token) + { + Load(p); + return ValueTask.CompletedTask; + } + + await Assert.ThrowsAsync(async () => await gate.RunAsync(probe, LoadAsync, Reset, ct)); + Assert.Equal(1, probe.Resets); + + probe.Fail = new InvalidDataException("bad table"); + await Assert.ThrowsAsync(async () => await gate.RunAsync(probe, LoadAsync, Reset, ct)); + probe.Fail = null; + await Assert.ThrowsAsync(async () => await gate.RunAsync(probe, LoadAsync, Reset, ct)); + Assert.Equal(2, probe.Loads); + } + + [Fact] + public async Task A_Sync_Caller_And_An_Async_Caller_Share_One_Load() + { + CancellationToken ct = TestContext.Current.CancellationToken; + using OnceGate gate = new(); + Probe probe = new(); + + static ValueTask LoadAsync(Probe p, CancellationToken token) + { + Load(p); + return ValueTask.CompletedTask; + } + + await gate.RunAsync(probe, LoadAsync, Reset, ct); + gate.Run(probe, Load, Reset); + Assert.Equal(1, probe.Loads); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Internal/ReaderLifetimeTests.cs b/tests/ExcelReader.Tests/Reader/Internal/ReaderLifetimeTests.cs new file mode 100644 index 00000000..ce3e685b --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Internal/ReaderLifetimeTests.cs @@ -0,0 +1,67 @@ +using ExcelReader.Core.Reader.Internal; + +namespace ExcelReader.Tests.Reader.Internal +{ + public sealed class ReaderLifetimeTests + { + [Fact] + public void Close_Alone_Releases() + { + int released = 0; + ReaderLifetime lifetime = new(() => released++); + lifetime.Close(); + Assert.Equal(1, released); + } + + [Fact] + public void Release_Happens_When_The_Last_Holder_Lets_Go() + { + int released = 0; + ReaderLifetime lifetime = new(() => released++); + lifetime.Acquire(this); + lifetime.Acquire(this); + + lifetime.Close(); + Assert.Equal(0, released); + lifetime.Release(); + Assert.Equal(0, released); + lifetime.Release(); + Assert.Equal(1, released); + } + + [Fact] + public void Close_Twice_Releases_Once() + { + int released = 0; + ReaderLifetime lifetime = new(() => released++); + lifetime.Close(); + lifetime.Close(); + Assert.Equal(1, released); + } + + [Fact] + public void Acquire_After_Close_Throws_Even_While_Holders_Remain() + { + ReaderLifetime lifetime = new(() => { }); + lifetime.Acquire(this); + lifetime.Close(); + ObjectDisposedException ex = Assert.Throws(() => lifetime.Acquire(this)); + Assert.Contains(nameof(ReaderLifetimeTests), ex.ObjectName, StringComparison.Ordinal); + } + + [Fact] + public void Concurrent_Holders_Release_Exactly_Once() + { + int released = 0; + ReaderLifetime lifetime = new(() => Interlocked.Increment(ref released)); + Parallel.For(0, 64, _ => + { + lifetime.Acquire(this); + lifetime.Release(); + }); + Assert.Equal(0, released); + lifetime.Close(); + Assert.Equal(1, released); + } + } +} From 1e49a2baa70edcdc5cb9a4421c48ecacec8ad472 Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Tue, 6 Oct 2026 10:46:12 -0300 Subject: [PATCH 18/28] feat(core): load XLSX shared strings once and keep the reader alive for its enumerators Co-Authored-By: Claude Sonnet 5.5 --- .../Reader/PooledStreamRowEnumerator.cs | 3 + .../Reader/Xlsx/XlsxReader.Enumerator.cs | 2 + .../Reader/Xlsx/XlsxReader.Loading.cs | 33 ++--- .../Reader/Xlsx/XlsxReader.cs | 60 +++++++-- .../Reader/Xlsx/XlsxLifetimeTests.cs | 118 ++++++++++++++++++ 5 files changed, 190 insertions(+), 26 deletions(-) create mode 100644 tests/ExcelReader.Tests/Reader/Xlsx/XlsxLifetimeTests.cs diff --git a/src/ExcelReader.Core/Reader/PooledStreamRowEnumerator.cs b/src/ExcelReader.Core/Reader/PooledStreamRowEnumerator.cs index 2cac3766..158eff56 100644 --- a/src/ExcelReader.Core/Reader/PooledStreamRowEnumerator.cs +++ b/src/ExcelReader.Core/Reader/PooledStreamRowEnumerator.cs @@ -14,6 +14,7 @@ public abstract class PooledStreamRowEnumerator private protected readonly BufferedStreamCursor _io; private protected readonly CellAccumulator _acc; private readonly bool _ownsSource; + private protected ReaderLifetime? _lease; private bool _deferred; private protected byte[] _buf => _io.Buf; private protected int _pos { get => _io.Pos; set => _io.Pos = value; } @@ -79,6 +80,7 @@ public void Dispose() } _source = null; ReturnBuffers(); + Interlocked.Exchange(ref _lease, null)?.Release(); } /// Asynchronous counterpart to . @@ -90,6 +92,7 @@ public async ValueTask DisposeAsync() } _source = null; ReturnBuffers(); + Interlocked.Exchange(ref _lease, null)?.Release(); } private protected void Fill() diff --git a/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Enumerator.cs b/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Enumerator.cs index b5db839d..9b54bf4d 100644 --- a/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Enumerator.cs +++ b/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Enumerator.cs @@ -38,6 +38,7 @@ internal Enumerator(XlsxReader reader, Stream sheet, long entryLength = 0, Cance _styleIsDate = reader._styleIsDate; _sharedOffsets = reader._sharedOffsets; _contentCache = reader._options.InternStrings ? new Utf8StringCache() : null; + _lease = reader.Lifetime; } internal Enumerator(XlsxReader reader, ZipEntryRef entry, CancellationToken ct) @@ -48,6 +49,7 @@ internal Enumerator(XlsxReader reader, ZipEntryRef entry, CancellationToken ct) _sharedOffsets = reader._sharedOffsets; _contentCache = reader._options.InternStrings ? new Utf8StringCache() : null; _entry = entry; + _lease = reader.Lifetime; } private protected override Stream OpenSource() diff --git a/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Loading.cs b/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Loading.cs index 06937c3e..c7de2bb8 100644 --- a/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Loading.cs +++ b/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Loading.cs @@ -70,11 +70,25 @@ private static bool ParseDate1904(ReadOnlySpan src) private void EnsureSharedLoaded() { - if (_sharedLoaded) + if (_sharedGate.IsDone) { return; } - _sharedLoaded = true; + _sharedGate.Run(this, static reader => reader.LoadShared(), static reader => reader.ReturnSharedTable()); + } + + private ValueTask EnsureSharedLoadedAsync(CancellationToken ct) + { + if (_sharedGate.IsDone) + { + return ValueTask.CompletedTask; + } + return _sharedGate.RunAsync( + this, static (reader, token) => reader.LoadSharedAsync(token), static reader => reader.ReturnSharedTable(), ct); + } + + private void LoadShared() + { if (!_zip.TryGetEntry("xl/sharedStrings.xml"u8, out ZipEntryRef entry)) { return; @@ -92,20 +106,11 @@ private void EnsureSharedLoaded() ParseSharedStreaming(stream, entry.UncompressedSize); } - private async ValueTask EnsureSharedLoadedAsync(CancellationToken ct) + private async ValueTask LoadSharedAsync(CancellationToken ct) { - if (_sharedLoaded) - { - return; - } - if (_zip.HasMemory) - { - EnsureSharedLoaded(); - return; - } - _sharedLoaded = true; - if (!_zip.TryGetEntry("xl/sharedStrings.xml"u8, out ZipEntryRef entry)) + if (_zip.HasMemory || !_zip.TryGetEntry("xl/sharedStrings.xml"u8, out ZipEntryRef entry)) { + LoadShared(); return; } WorkbookLookups.ThrowIfSharedEntryTooLarge(entry.UncompressedSize, _decompressedBytes, _options); diff --git a/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.cs b/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.cs index 0c124f96..65f65499 100644 --- a/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.cs +++ b/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.cs @@ -1,4 +1,5 @@ using System.Buffers; +using System.Diagnostics.CodeAnalysis; using ExcelReader.Core.Reader.Internal; using ExcelReader.Core.Reader.Sources; using ExcelReader.Core.Reader.Zip; @@ -15,11 +16,14 @@ public sealed partial class XlsxReader : IExcelRowReader, IExcelRowReader public Enumerator GetEnumerator() { - EnsureSharedLoaded(); - ZipEntryRef entry = WorkbookLookups.GetWorksheetEntry(_zip, _sheets[_current].Path); - return new Enumerator(this, _zip.OpenEntryStream(entry, _decompressedBytes, _options), entry.UncompressedSize); + Lifetime.Acquire(this); + try + { + EnsureSharedLoaded(); + ZipEntryRef entry = WorkbookLookups.GetWorksheetEntry(_zip, _sheets[_current].Path); + return new Enumerator(this, _zip.OpenEntryStream(entry, _decompressedBytes, _options), entry.UncompressedSize); + } + catch + { + Lifetime.Release(); + throw; + } } IExcelRowEnumerator IExcelRowReader.GetEnumerator() @@ -169,7 +184,16 @@ public Enumerator GetAsyncEnumerator(CancellationToken ct = default) { return GetEnumerator(); } - return new Enumerator(this, WorkbookLookups.GetWorksheetEntry(_zip, _sheets[_current].Path), ct); + Lifetime.Acquire(this); + try + { + return new Enumerator(this, WorkbookLookups.GetWorksheetEntry(_zip, _sheets[_current].Path), ct); + } + catch + { + Lifetime.Release(); + throw; + } } IExcelRowEnumerator IExcelRowReader.GetAsyncEnumerator(CancellationToken ct) @@ -190,19 +214,31 @@ IExcelRowEnumerator IExcelRowReader.GetAsyncEnumerator(Canc /// public void Dispose() { - if (_sharedFlat.Length > 0) - { - ArrayPool.Shared.Return(_sharedFlat); - _sharedFlat = []; - } - _zip.Dispose(); + Lifetime.Close(); } /// public ValueTask DisposeAsync() { - Dispose(); + Lifetime.Close(); return ValueTask.CompletedTask; } + + private void ReleaseResources() + { + ReturnSharedTable(); + _sharedGate.Dispose(); + _zip.Dispose(); + } + + private void ReturnSharedTable() + { + if (_sharedFlat.Length > 0) + { + ArrayPool.Shared.Return(_sharedFlat); + } + _sharedFlat = []; + _sharedOffsets = [0]; + } } } diff --git a/tests/ExcelReader.Tests/Reader/Xlsx/XlsxLifetimeTests.cs b/tests/ExcelReader.Tests/Reader/Xlsx/XlsxLifetimeTests.cs new file mode 100644 index 00000000..0b54470f --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Xlsx/XlsxLifetimeTests.cs @@ -0,0 +1,118 @@ +using System.Globalization; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Xlsx; +using ExcelReader.Core.Writer.Xlsx; + +namespace ExcelReader.Tests.Reader.Xlsx +{ + public sealed class XlsxLifetimeTests + { + private static byte[] BuildXlsx(int rows = 300) + { + using MemoryStream buffer = new(); + using (XlsxWorkbookWriter workbook = XlsxWorkbookWriter.Create(buffer, leaveOpen: true, new XlsxWriterOptions { UseSharedStrings = true })) + { + XlsxSheetWriter sheet = workbook.AddSheet("s"); + for (int r = 0; r < rows; r++) + { + using XlsxRowWriter row = sheet.StartRow(); + row.Write("name-" + (r % 40).ToString(CultureInfo.InvariantCulture)); + row.Write(r); + } + sheet.End(); + sheet.Dispose(); + workbook.End(); + } + return buffer.ToArray(); + } + + private static List Drain(XlsxReader.Enumerator e) + { + List values = []; + while (e.MoveNext()) + { + values.Add(e.Current[0].GetString()); + } + return values; + } + + [Fact] + public void A_Failed_Shared_String_Load_Fails_Every_Later_Enumeration() + { + using XlsxReader reader = Excel.FromXlsx(BuildXlsx(), new ExcelReaderOptions { MaxSharedStringBytes = 16 }); + Assert.Throws(() => reader.GetEnumerator()); + Assert.Throws(() => reader.GetEnumerator()); + } + + [Fact] + public async Task A_Failed_Shared_String_Load_Fails_Every_Later_Async_Enumeration() + { + using XlsxReader reader = Excel.FromXlsx( + new MemoryStream(BuildXlsx(), writable: false), leaveOpen: false, new ExcelReaderOptions { MaxSharedStringBytes = 16 }); + for (int attempt = 0; attempt < 2; attempt++) + { + await using XlsxReader.Enumerator e = reader.GetAsyncEnumerator(TestContext.Current.CancellationToken); + await Assert.ThrowsAsync(async () => await e.MoveNextAsync()); + } + } + + [Fact] + public void GetEnumerator_After_Dispose_Throws() + { + XlsxReader reader = Excel.FromXlsx(BuildXlsx()); + reader.Dispose(); + Assert.Throws(() => reader.GetEnumerator()); + Assert.Throws(() => reader.GetAsyncEnumerator(TestContext.Current.CancellationToken)); + } + + [Fact] + public void An_Enumerator_Outlives_Its_Reader() + { + byte[] bytes = BuildXlsx(); + List expected; + using (XlsxReader reference = Excel.FromXlsx(bytes)) + { + using XlsxReader.Enumerator all = reference.GetEnumerator(); + expected = Drain(all); + } + + TrickleStream stream = new(bytes); + XlsxReader reader = Excel.FromXlsx(stream, leaveOpen: false); + XlsxReader.Enumerator e = reader.GetEnumerator(); + Assert.True(e.MoveNext()); + + reader.Dispose(); + Assert.True(stream.CanRead); + + List rest = Drain(e); + Assert.Equal(expected.Skip(1), rest, StringComparer.Ordinal); + + e.Dispose(); + Assert.False(stream.CanRead); + } + + [Fact] + public void Disposing_Reader_And_Enumerator_Twice_Is_Harmless() + { + TrickleStream stream = new(BuildXlsx(rows: 20)); + XlsxReader reader = Excel.FromXlsx(stream, leaveOpen: false); + XlsxReader.Enumerator e = reader.GetEnumerator(); + e.Dispose(); + e.Dispose(); + Assert.True(stream.CanRead); + reader.Dispose(); + reader.Dispose(); + Assert.False(stream.CanRead); + } + + [Fact] + public void A_Failed_GetEnumerator_Does_Not_Keep_The_Reader_Alive() + { + TrickleStream stream = new(BuildXlsx(rows: 20)); + XlsxReader reader = Excel.FromXlsx(stream, leaveOpen: false, new ExcelReaderOptions { MaxSharedStringBytes = 16 }); + Assert.Throws(() => reader.GetEnumerator()); + reader.Dispose(); + Assert.False(stream.CanRead); + } + } +} From fecefc4c8d7193e86ab514712c4373ab2d76e802 Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Tue, 6 Oct 2026 10:52:30 -0300 Subject: [PATCH 19/28] feat(core): keep an XLSB reader alive for its enumerators Co-Authored-By: Claude Sonnet 5.5 --- .../Reader/Xlsb/XlsbReader.Enumerator.cs | 2 + .../Reader/Xlsb/XlsbReader.cs | 50 ++++++++--- .../Reader/Xlsb/XlsbLifetimeTests.cs | 88 +++++++++++++++++++ 3 files changed, 126 insertions(+), 14 deletions(-) create mode 100644 tests/ExcelReader.Tests/Reader/Xlsb/XlsbLifetimeTests.cs diff --git a/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Enumerator.cs b/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Enumerator.cs index 69deabae..957cb697 100644 --- a/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Enumerator.cs +++ b/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Enumerator.cs @@ -35,6 +35,7 @@ internal Enumerator(XlsbReader reader, Stream sheet, long entryLength = 0, Cance _styleIsDate = reader._styleIsDate; _sharedOffsets = reader._sharedOffsets; _contentCache = reader._options.InternStrings ? new Utf8StringCache() : null; + _lease = reader.Lifetime; } internal Enumerator(XlsbReader reader, ZipEntryRef entry, CancellationToken ct) @@ -45,6 +46,7 @@ internal Enumerator(XlsbReader reader, ZipEntryRef entry, CancellationToken ct) _sharedOffsets = reader._sharedOffsets; _contentCache = reader._options.InternStrings ? new Utf8StringCache() : null; _entry = entry; + _lease = reader.Lifetime; } private protected override Stream OpenSource() diff --git a/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.cs b/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.cs index c84b0ed0..e46b52d2 100644 --- a/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.cs +++ b/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.cs @@ -24,10 +24,11 @@ public sealed partial class XlsbReader : IExcelRowReader, IExcelRowReader public Enumerator GetEnumerator() { - ZipEntryRef entry = WorkbookLookups.GetWorksheetEntry(_zip!, _sheets![_current].Path); - return new Enumerator(this, _zip!.OpenEntryStream(entry, _decompressedBytes, _options), entry.UncompressedSize); + Lifetime.Acquire(this); + try + { + ZipEntryRef entry = WorkbookLookups.GetWorksheetEntry(_zip!, _sheets![_current].Path); + return new Enumerator(this, _zip!.OpenEntryStream(entry, _decompressedBytes, _options), entry.UncompressedSize); + } + catch + { + Lifetime.Release(); + throw; + } } IExcelRowEnumerator IExcelRowReader.GetEnumerator() @@ -239,7 +251,16 @@ public Enumerator GetAsyncEnumerator(CancellationToken ct = default) { return GetEnumerator(); } - return new Enumerator(this, WorkbookLookups.GetWorksheetEntry(_zip, _sheets![_current].Path), ct); + Lifetime.Acquire(this); + try + { + return new Enumerator(this, WorkbookLookups.GetWorksheetEntry(_zip, _sheets![_current].Path), ct); + } + catch + { + Lifetime.Release(); + throw; + } } IExcelRowEnumerator IExcelRowReader.GetAsyncEnumerator(CancellationToken ct) @@ -251,23 +272,24 @@ IExcelRowEnumerator IExcelRowReader.GetAsyncEnumerator(Canc /// public void Dispose() { - if (Interlocked.Exchange(ref _disposed, 1) != 0) - { - return; - } - if (_pooledSharedFlat) - { - ArrayPool.Shared.Return(_sharedFlat); - } - _zip?.Dispose(); + Lifetime.Close(); } /// public ValueTask DisposeAsync() { - Dispose(); + Lifetime.Close(); return ValueTask.CompletedTask; } + private void ReleaseResources() + { + if (_pooledSharedFlat) + { + ArrayPool.Shared.Return(_sharedFlat); + } + _zip?.Dispose(); + } + } } diff --git a/tests/ExcelReader.Tests/Reader/Xlsb/XlsbLifetimeTests.cs b/tests/ExcelReader.Tests/Reader/Xlsb/XlsbLifetimeTests.cs new file mode 100644 index 00000000..f409132a --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Xlsb/XlsbLifetimeTests.cs @@ -0,0 +1,88 @@ +using System.Globalization; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Xlsb; +using ExcelReader.Core.Writer.Xlsb; + +namespace ExcelReader.Tests.Reader.Xlsb +{ + public sealed class XlsbLifetimeTests + { + private static byte[] BuildXlsb(int rows = 300) + { + using MemoryStream buffer = new(); + using (XlsbWorkbookWriter workbook = XlsbWorkbookWriter.Create(buffer, leaveOpen: true, new XlsbWriterOptions { UseSharedStrings = true })) + { + XlsbSheetWriter sheet = workbook.AddSheet("s"); + for (int r = 0; r < rows; r++) + { + using XlsbRowWriter row = sheet.StartRow(); + row.Write("name-" + (r % 40).ToString(CultureInfo.InvariantCulture)); + row.Write(r); + } + sheet.End(); + sheet.Dispose(); + workbook.End(); + } + return buffer.ToArray(); + } + + private static List Drain(XlsbReader.Enumerator e) + { + List values = []; + while (e.MoveNext()) + { + values.Add(e.Current[0].GetString()); + } + return values; + } + + [Fact] + public void GetEnumerator_After_Dispose_Throws() + { + XlsbReader reader = Excel.FromXlsb(BuildXlsb()); + reader.Dispose(); + Assert.Throws(() => reader.GetEnumerator()); + Assert.Throws(() => reader.GetAsyncEnumerator(TestContext.Current.CancellationToken)); + } + + [Fact] + public void An_Enumerator_Outlives_Its_Reader() + { + byte[] bytes = BuildXlsb(); + List expected; + using (XlsbReader reference = Excel.FromXlsb(bytes)) + { + using XlsbReader.Enumerator all = reference.GetEnumerator(); + expected = Drain(all); + } + + TrickleStream stream = new(bytes); + XlsbReader reader = Excel.FromXlsb(stream, leaveOpen: false); + XlsbReader.Enumerator e = reader.GetEnumerator(); + Assert.True(e.MoveNext()); + + reader.Dispose(); + Assert.True(stream.CanRead); + + List rest = Drain(e); + Assert.Equal(expected.Skip(1), rest, StringComparer.Ordinal); + + e.Dispose(); + Assert.False(stream.CanRead); + } + + [Fact] + public void Disposing_Reader_And_Enumerator_Twice_Is_Harmless() + { + TrickleStream stream = new(BuildXlsb(rows: 20)); + XlsbReader reader = Excel.FromXlsb(stream, leaveOpen: false); + XlsbReader.Enumerator e = reader.GetEnumerator(); + e.Dispose(); + e.Dispose(); + Assert.True(stream.CanRead); + reader.Dispose(); + reader.Dispose(); + Assert.False(stream.CanRead); + } + } +} From c8dbaa6babf8762a8e3ef066927450acabc4c2fc Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Tue, 6 Oct 2026 10:58:44 -0300 Subject: [PATCH 20/28] feat(core): keep an XLS reader alive for its enumerators Co-Authored-By: Claude Sonnet 5.5 --- .../Reader/Xls/XlsReader.Enumerator.cs | 3 + src/ExcelReader.Core/Reader/Xls/XlsReader.cs | 40 +++++++-- .../Reader/Xls/XlsLifetimeTests.cs | 88 +++++++++++++++++++ 3 files changed, 124 insertions(+), 7 deletions(-) create mode 100644 tests/ExcelReader.Tests/Reader/Xls/XlsLifetimeTests.cs diff --git a/src/ExcelReader.Core/Reader/Xls/XlsReader.Enumerator.cs b/src/ExcelReader.Core/Reader/Xls/XlsReader.Enumerator.cs index 6328efa3..2f8cabcd 100644 --- a/src/ExcelReader.Core/Reader/Xls/XlsReader.Enumerator.cs +++ b/src/ExcelReader.Core/Reader/Xls/XlsReader.Enumerator.cs @@ -22,6 +22,7 @@ public sealed class Enumerator : IExcelRowEnumerator private bool _emptyRow; private int _row; private int _rowFloor = -1; + private ReaderLifetime? _lease; internal Enumerator(XlsReader reader, int sheetOffset, CancellationToken ct = default) { @@ -31,6 +32,7 @@ internal Enumerator(XlsReader reader, int sheetOffset, CancellationToken ct = de _row = -1; _acc = new CellAccumulator(reader._options.MaxCellBytes, nameof(ExcelReaderOptions.MaxCellBytes)); _contentCache = reader._options.InternStrings ? new Utf8StringCache() : null; + _lease = reader.Lifetime; } /// @@ -332,6 +334,7 @@ private void ReturnBuffers() { _cursor.Dispose(); _acc.Return(); + Interlocked.Exchange(ref _lease, null)?.Release(); } } } diff --git a/src/ExcelReader.Core/Reader/Xls/XlsReader.cs b/src/ExcelReader.Core/Reader/Xls/XlsReader.cs index 15ed04fe..89ef3229 100644 --- a/src/ExcelReader.Core/Reader/Xls/XlsReader.cs +++ b/src/ExcelReader.Core/Reader/Xls/XlsReader.cs @@ -18,6 +18,8 @@ public sealed partial class XlsReader : IExcelRowReader, IExcelRowReader data, ExcelReaderOptions? options = null private XlsReader(WorkbookStream workbook, ExcelReaderOptions? options = null) { + Lifetime = new ReaderLifetime(ReleaseResources); _workbook = workbook; _options = options ?? ExcelReaderOptions.Default; using (BiffCursor cursor = workbook.OpenCursor()) @@ -123,7 +126,16 @@ public void MoveToSheet(int index) /// public Enumerator GetEnumerator() { - return new Enumerator(this, _sheets[_current].Offset); + Lifetime.Acquire(this); + try + { + return new Enumerator(this, _sheets[_current].Offset); + } + catch + { + Lifetime.Release(); + throw; + } } IExcelRowEnumerator IExcelRowReader.GetEnumerator() @@ -136,7 +148,16 @@ IExcelRowEnumerator IExcelRowReader.GetEnumerator() public Enumerator GetAsyncEnumerator(CancellationToken ct = default) { ct.ThrowIfCancellationRequested(); - return new Enumerator(this, _sheets[_current].Offset, ct); + Lifetime.Acquire(this); + try + { + return new Enumerator(this, _sheets[_current].Offset, ct); + } + catch + { + Lifetime.Release(); + throw; + } } IExcelRowEnumerator IExcelRowReader.GetAsyncEnumerator(CancellationToken ct) @@ -147,19 +168,24 @@ IExcelRowEnumerator IExcelRowReader.GetAsyncEnumerator(Canc /// public void Dispose() { - _workbook.Dispose(); - _sharedFlat = []; - _sharedOffsets = [0]; - _sharedStringCache = null; + Lifetime.Close(); } /// public ValueTask DisposeAsync() { - Dispose(); + Lifetime.Close(); return ValueTask.CompletedTask; } + private void ReleaseResources() + { + _workbook.Dispose(); + _sharedFlat = []; + _sharedOffsets = [0]; + _sharedStringCache = null; + } + private static void ParseWorkbookGlobals( BiffCursor cursor, ExcelReaderOptions options, diff --git a/tests/ExcelReader.Tests/Reader/Xls/XlsLifetimeTests.cs b/tests/ExcelReader.Tests/Reader/Xls/XlsLifetimeTests.cs new file mode 100644 index 00000000..c4f31eaa --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Xls/XlsLifetimeTests.cs @@ -0,0 +1,88 @@ +using System.Globalization; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Xls; +using ExcelReader.Core.Writer.Xls; + +namespace ExcelReader.Tests.Reader.Xls +{ + public sealed class XlsLifetimeTests + { + private static byte[] BuildXls(int rows = 300) + { + using MemoryStream buffer = new(); + using (XlsWorkbookWriter workbook = XlsWorkbookWriter.Create(buffer, leaveOpen: true)) + { + XlsSheetWriter sheet = workbook.AddSheet("s"); + for (int r = 0; r < rows; r++) + { + using XlsRowWriter row = sheet.StartRow(); + row.Write("name-" + (r % 40).ToString(CultureInfo.InvariantCulture)); + row.Write(r); + } + sheet.End(); + sheet.Dispose(); + workbook.End(); + } + return buffer.ToArray(); + } + + private static List Drain(XlsReader.Enumerator e) + { + List values = []; + while (e.MoveNext()) + { + values.Add(e.Current[0].GetString()); + } + return values; + } + + [Fact] + public void GetEnumerator_After_Dispose_Throws() + { + XlsReader reader = Excel.FromXls(BuildXls()); + reader.Dispose(); + Assert.Throws(() => reader.GetEnumerator()); + Assert.Throws(() => reader.GetAsyncEnumerator(TestContext.Current.CancellationToken)); + } + + [Fact] + public void An_Enumerator_Outlives_Its_Reader() + { + byte[] bytes = BuildXls(); + List expected; + using (XlsReader reference = Excel.FromXls(bytes)) + { + using XlsReader.Enumerator all = reference.GetEnumerator(); + expected = Drain(all); + } + + TrickleStream stream = new(bytes); + XlsReader reader = Excel.FromXls(stream, leaveOpen: false); + XlsReader.Enumerator e = reader.GetEnumerator(); + Assert.True(e.MoveNext()); + + reader.Dispose(); + Assert.True(stream.CanRead); + + List rest = Drain(e); + Assert.Equal(expected.Skip(1), rest, StringComparer.Ordinal); + + e.Dispose(); + Assert.False(stream.CanRead); + } + + [Fact] + public void Disposing_Reader_And_Enumerator_Twice_Is_Harmless() + { + TrickleStream stream = new(BuildXls(rows: 20)); + XlsReader reader = Excel.FromXls(stream, leaveOpen: false); + XlsReader.Enumerator e = reader.GetEnumerator(); + e.Dispose(); + e.Dispose(); + Assert.True(stream.CanRead); + reader.Dispose(); + reader.Dispose(); + Assert.False(stream.CanRead); + } + } +} From 5f745f1312c919da5fe72a2d4fe237aff71f3894 Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Tue, 6 Oct 2026 11:06:04 -0300 Subject: [PATCH 21/28] fix(core): refuse use after dispose in the sources and reject negative offsets everywhere ByteSourceStream and ZipIndex throw ObjectDisposedException after Dispose; every ByteSource rejects a negative offset with ArgumentOutOfRangeException. Co-Authored-By: Claude Sonnet 5.5 --- .../Reader/Sources/ByteSourceStream.cs | 4 ++ .../Reader/Sources/FileByteSource.cs | 2 + .../Reader/Sources/MemoryByteSource.cs | 4 +- .../Reader/Sources/StreamByteSource.cs | 2 + src/ExcelReader.Core/Reader/Zip/ZipIndex.cs | 10 ++++ .../Reader/Sources/ByteSourceStreamTests.cs | 32 +++++++++++++ .../Reader/Sources/ByteSourceTests.cs | 46 +++++++++++++++++++ .../Reader/Zip/ZipIndexSourceTests.cs | 33 +++++++++++++ 8 files changed, 132 insertions(+), 1 deletion(-) diff --git a/src/ExcelReader.Core/Reader/Sources/ByteSourceStream.cs b/src/ExcelReader.Core/Reader/Sources/ByteSourceStream.cs index 2d798bd0..fb79e43e 100644 --- a/src/ExcelReader.Core/Reader/Sources/ByteSourceStream.cs +++ b/src/ExcelReader.Core/Reader/Sources/ByteSourceStream.cs @@ -13,6 +13,7 @@ internal sealed class ByteSourceStream : Stream private byte[]? _buffer; private int _bufferPos; private int _bufferLen; + private bool _disposed; internal ByteSourceStream(ByteSource source, long offset, long length) { @@ -42,6 +43,7 @@ public override int Read(byte[] buffer, int offset, int count) public override int Read(Span destination) { + ObjectDisposedException.ThrowIf(_disposed, this); if (destination.IsEmpty) { return 0; @@ -73,6 +75,7 @@ public override int Read(Span destination) public override async ValueTask ReadAsync(Memory destination, CancellationToken cancellationToken = default) { + ObjectDisposedException.ThrowIf(_disposed, this); if (destination.IsEmpty) { return 0; @@ -138,6 +141,7 @@ public override void Write(byte[] buffer, int offset, int count) protected override void Dispose(bool disposing) { + _disposed = true; byte[]? buffer = Interlocked.Exchange(ref _buffer, null); if (buffer is not null) { diff --git a/src/ExcelReader.Core/Reader/Sources/FileByteSource.cs b/src/ExcelReader.Core/Reader/Sources/FileByteSource.cs index 7a3c54dc..ebd9f584 100644 --- a/src/ExcelReader.Core/Reader/Sources/FileByteSource.cs +++ b/src/ExcelReader.Core/Reader/Sources/FileByteSource.cs @@ -18,11 +18,13 @@ internal FileByteSource(SafeFileHandle handle, long length, IDisposable? owner) internal override int Read(long offset, Span destination) { + ArgumentOutOfRangeException.ThrowIfNegative(offset); return offset >= Length ? 0 : RandomAccess.Read(_handle, destination, offset); } internal override ValueTask ReadAsync(long offset, Memory destination, CancellationToken ct) { + ArgumentOutOfRangeException.ThrowIfNegative(offset); return offset >= Length ? new ValueTask(0) : RandomAccess.ReadAsync(_handle, destination, offset, ct); } diff --git a/src/ExcelReader.Core/Reader/Sources/MemoryByteSource.cs b/src/ExcelReader.Core/Reader/Sources/MemoryByteSource.cs index bbf81382..68a66d85 100644 --- a/src/ExcelReader.Core/Reader/Sources/MemoryByteSource.cs +++ b/src/ExcelReader.Core/Reader/Sources/MemoryByteSource.cs @@ -15,7 +15,8 @@ internal MemoryByteSource(ReadOnlyMemory memory, IDisposable? owner) internal override int Read(long offset, Span destination) { - if (offset < 0 || offset >= _memory.Length) + ArgumentOutOfRangeException.ThrowIfNegative(offset); + if (offset >= _memory.Length) { return 0; } @@ -26,6 +27,7 @@ internal override int Read(long offset, Span destination) internal override ValueTask ReadAsync(long offset, Memory destination, CancellationToken ct) { + ArgumentOutOfRangeException.ThrowIfNegative(offset); return new ValueTask(Read(offset, destination.Span)); } diff --git a/src/ExcelReader.Core/Reader/Sources/StreamByteSource.cs b/src/ExcelReader.Core/Reader/Sources/StreamByteSource.cs index 0a31af0e..d8c6bc61 100644 --- a/src/ExcelReader.Core/Reader/Sources/StreamByteSource.cs +++ b/src/ExcelReader.Core/Reader/Sources/StreamByteSource.cs @@ -21,6 +21,7 @@ internal StreamByteSource(Stream stream, bool leaveOpen) internal override int Read(long offset, Span destination) { + ArgumentOutOfRangeException.ThrowIfNegative(offset); if (offset >= Length) { return 0; @@ -39,6 +40,7 @@ internal override int Read(long offset, Span destination) internal override async ValueTask ReadAsync(long offset, Memory destination, CancellationToken ct) { + ArgumentOutOfRangeException.ThrowIfNegative(offset); if (offset >= Length) { return 0; diff --git a/src/ExcelReader.Core/Reader/Zip/ZipIndex.cs b/src/ExcelReader.Core/Reader/Zip/ZipIndex.cs index 498c6f74..cb6b225d 100644 --- a/src/ExcelReader.Core/Reader/Zip/ZipIndex.cs +++ b/src/ExcelReader.Core/Reader/Zip/ZipIndex.cs @@ -284,8 +284,14 @@ private static void ThrowIfDuplicateEntryNames(ReadOnlyMemory directory, Z } } + private void ThrowIfDisposed() + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); + } + internal bool TryGetEntry(ReadOnlySpan utf8Name, out ZipEntryRef entry) { + ThrowIfDisposed(); ReadOnlySpan directory = _directory.Span; foreach (ref readonly ZipEntryRef candidate in _entries.AsSpan(0, Count)) { @@ -301,6 +307,7 @@ internal bool TryGetEntry(ReadOnlySpan utf8Name, out ZipEntryRef entry) internal ZipPart OpenPart(in ZipEntryRef entry, DecompressedByteCounter counter, string entryLimitName = "", long entryLimit = 0) { + ThrowIfDisposed(); ThrowIfPartTooLarge(entry, counter, entryLimitName, entryLimit); long dataOffset = ResolveDataOffset(entry); ThrowIfEntryDataInvalid(dataOffset, entry); @@ -321,6 +328,7 @@ internal ZipPart OpenPart(in ZipEntryRef entry, DecompressedByteCounter counter, internal async ValueTask OpenPartAsync( ZipEntryRef entry, DecompressedByteCounter counter, CancellationToken ct, string entryLimitName = "", long entryLimit = 0) { + ThrowIfDisposed(); if (HasMemory) { return OpenPart(entry, counter, entryLimitName, entryLimit); @@ -372,6 +380,7 @@ internal ZipPart OpenPartOrDefault(ReadOnlySpan utf8Name, DecompressedByte internal LimitedReadStream OpenEntryStream( in ZipEntryRef entry, DecompressedByteCounter counter, ExcelReaderOptions options, string entryLimitName = "", long entryLimit = 0) { + ThrowIfDisposed(); long dataOffset = ResolveDataOffset(entry); return WrapEntryStream(entry, dataOffset, counter, options, entryLimitName, entryLimit); } @@ -380,6 +389,7 @@ internal async ValueTask OpenEntryStreamAsync( ZipEntryRef entry, DecompressedByteCounter counter, ExcelReaderOptions options, CancellationToken ct, string entryLimitName = "", long entryLimit = 0) { + ThrowIfDisposed(); long dataOffset = HasMemory ? ResolveDataOffset(entry) : await ResolveDataOffsetAsync(entry, ct).ConfigureAwait(false); return WrapEntryStream(entry, dataOffset, counter, options, entryLimitName, entryLimit); } diff --git a/tests/ExcelReader.Tests/Reader/Sources/ByteSourceStreamTests.cs b/tests/ExcelReader.Tests/Reader/Sources/ByteSourceStreamTests.cs index bc8c2bdb..ed68c535 100644 --- a/tests/ExcelReader.Tests/Reader/Sources/ByteSourceStreamTests.cs +++ b/tests/ExcelReader.Tests/Reader/Sources/ByteSourceStreamTests.cs @@ -109,5 +109,37 @@ public void Dispose_Does_Not_Dispose_The_Source() new ByteSourceStream(source, 0, 10).Dispose(); Assert.False(source.Disposed); } + + [Fact] + public void Read_After_Dispose_Throws() + { + using CountingSource source = new(Payload(1000)); + ByteSourceStream stream = new(source, 0, 1000); + Assert.Equal(10, stream.Read(new byte[10])); + stream.Dispose(); + Assert.Throws(() => stream.Read(new byte[10])); + } + + [Fact] + public async Task ReadAsync_After_Dispose_Throws() + { + using CountingSource source = new(Payload(1000)); + ByteSourceStream stream = new(source, 0, 1000); + await stream.DisposeAsync(); + await Assert.ThrowsAsync( + async () => await stream.ReadAsync(new byte[10], TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task A_Large_Async_Read_Bypasses_The_Buffer() + { + byte[] bytes = Payload(300_000); + using CountingSource source = new(bytes); + await using ByteSourceStream stream = new(source, 0, 300_000); + byte[] big = new byte[300_000]; + Assert.Equal(300_000, await stream.ReadAsync(big, TestContext.Current.CancellationToken)); + Assert.Equal(1, source.Reads); + Assert.True(big.AsSpan().SequenceEqual(bytes)); + } } } diff --git a/tests/ExcelReader.Tests/Reader/Sources/ByteSourceTests.cs b/tests/ExcelReader.Tests/Reader/Sources/ByteSourceTests.cs index b60e88f7..d9a0135c 100644 --- a/tests/ExcelReader.Tests/Reader/Sources/ByteSourceTests.cs +++ b/tests/ExcelReader.Tests/Reader/Sources/ByteSourceTests.cs @@ -222,5 +222,51 @@ public void A_MemoryStream_Subclass_Is_Read_Through_Its_Own_Read() Assert.Equal((byte)(Payload[100 + i] ^ 0xFF), read[i]); } } + + [Theory] + [MemberData(nameof(Kinds))] + public void A_Negative_Offset_Is_Rejected(Kind kind) + { + using ByteSource source = Open(kind); + Assert.Throws(() => source.Read(-1, new byte[4])); + } + + [Theory] + [MemberData(nameof(Kinds))] + public async Task ReadAsync_At_The_End_Returns_Zero(Kind kind) + { + using ByteSource source = Open(kind); + Assert.Equal(0, await source.ReadAsync(Payload.Length, new byte[16], TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task A_Cancelled_Token_Stops_A_Stream_Read() + { + using ByteSource source = ByteSource.FromStream(new TrickleStream(Payload), leaveOpen: false); + using CancellationTokenSource cancelled = new(); + await cancelled.CancelAsync(); + await Assert.ThrowsAnyAsync( + async () => await source.ReadAsync(0, new byte[16], cancelled.Token)); + } + + [Fact] + public void An_Exposed_MemoryStream_With_A_Buffer_Offset_Reads_From_Its_Origin() + { + byte[] backing = new byte[Payload.Length + 100]; + Payload.CopyTo(backing, 100); + using ByteSource source = ByteSource.FromStream( + new MemoryStream(backing, 100, Payload.Length, writable: false, publiclyVisible: true), leaveOpen: false); + byte[] read = new byte[64]; + source.ReadExactly(10, read); + Assert.True(read.AsSpan().SequenceEqual(Payload.AsSpan(10, 64))); + } + + [Fact] + public async Task FromStreamAsync_Leaves_A_Borrowed_NonSeekable_Stream_Open() + { + using NonSeekableStream stream = new(Payload); + using ByteSource source = await ByteSource.FromStreamAsync(stream, leaveOpen: true, TestContext.Current.CancellationToken); + Assert.Equal(-1, stream.ReadByte()); + } } } diff --git a/tests/ExcelReader.Tests/Reader/Zip/ZipIndexSourceTests.cs b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexSourceTests.cs index 1c24085e..c3f64a0c 100644 --- a/tests/ExcelReader.Tests/Reader/Zip/ZipIndexSourceTests.cs +++ b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexSourceTests.cs @@ -437,5 +437,38 @@ public void Test_Workbooks_Match_ZipArchive_Entry_For_Entry() } Assert.True(checkedFiles > 0, "no ZIP-based workbook found under " + dataDirectory); } + + [Theory] + [MemberData(nameof(Kinds))] + public void A_Disposed_Index_Refuses_Every_Open(Kind kind) + { + ZipIndex index = ZipIndex.Create(Open(kind, BuildZip(dataDescriptors: false, scale: 20)), ExcelReaderOptions.Default); + Assert.True(index.TryGetEntry(System.Text.Encoding.UTF8.GetBytes(Names[0]), out ZipEntryRef entry)); + index.Dispose(); + + Assert.Throws(() => index.TryGetEntry(System.Text.Encoding.UTF8.GetBytes(Names[0]), out _)); + Assert.Throws(() => index.OpenPart(entry, Counter())); + Assert.Throws(() => index.OpenEntryStream(entry, Counter(), ExcelReaderOptions.Default)); + } + + [Fact] + public async Task A_Failed_CreateAsync_Disposes_The_Source() + { + TrickleStream stream = new(new byte[100]); + await Assert.ThrowsAsync(async () => await ZipIndex.CreateAsync( + ByteSource.FromStream(stream, leaveOpen: false), ExcelReaderOptions.Default, TestContext.Current.CancellationToken)); + Assert.False(stream.CanRead); + } + + [Fact] + public async Task A_Cancelled_CreateAsync_Disposes_The_Source() + { + TrickleStream stream = new(BuildZip(dataDescriptors: false, scale: 20)); + using CancellationTokenSource cancelled = new(); + await cancelled.CancelAsync(); + await Assert.ThrowsAnyAsync(async () => await ZipIndex.CreateAsync( + ByteSource.FromStream(stream, leaveOpen: false), ExcelReaderOptions.Default, cancelled.Token)); + Assert.False(stream.CanRead); + } } } From cee9ea5f143cb85fba50b6a1084f3786f1b0bc08 Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Tue, 6 Oct 2026 11:12:22 -0300 Subject: [PATCH 22/28] fix(core): dispose the row enumerator when the ExcelDataReader constructor throws The constructor reads the header row before it returns. A throw there (a limit, a corrupt sheet) dropped the enumerator it had just taken, and that enumerator's lease kept the reader's source open after the caller disposed the reader. Co-Authored-By: Claude Opus 5.5 --- .../Reader/ExcelDataReader.cs | 30 +++++++++++------- .../Reader/ExcelDataReaderTests.cs | 31 +++++++++++++++++++ 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/src/ExcelReader.Core/Reader/ExcelDataReader.cs b/src/ExcelReader.Core/Reader/ExcelDataReader.cs index 7d965b00..7ec63e4c 100644 --- a/src/ExcelReader.Core/Reader/ExcelDataReader.cs +++ b/src/ExcelReader.Core/Reader/ExcelDataReader.cs @@ -58,21 +58,29 @@ public ExcelDataReader(IExcelRowReader reader, int headerRow = 1) _sheetName = reader.SheetName; _rows = reader.GetEnumerator(); _pendingConsumed = true; - if (headerRow > 0 && SchemaInference.TrySkipToHeaderRow(_rows, headerRow, out _)) + try { - _names = ReadHeaderNames(_rows.Current); - } - else if (headerRow > 0) - { - _names = []; + if (headerRow > 0 && SchemaInference.TrySkipToHeaderRow(_rows, headerRow, out _)) + { + _names = ReadHeaderNames(_rows.Current); + } + else if (headerRow > 0) + { + _names = []; + } + else + { + _hasPendingRow = _rows.MoveNext(); + _pendingConsumed = false; + _names = _hasPendingRow ? new string?[_rows.Current.ColumnCount] : []; + } + _ordinals = BuildOrdinals(_names); } - else + catch { - _hasPendingRow = _rows.MoveNext(); - _pendingConsumed = false; - _names = _hasPendingRow ? new string?[_rows.Current.ColumnCount] : []; + _rows.Dispose(); + throw; } - _ordinals = BuildOrdinals(_names); } private static string?[] ReadHeaderNames(Row header) diff --git a/tests/ExcelReader.Tests/Reader/ExcelDataReaderTests.cs b/tests/ExcelReader.Tests/Reader/ExcelDataReaderTests.cs index 6fef2f24..c8bf18c3 100644 --- a/tests/ExcelReader.Tests/Reader/ExcelDataReaderTests.cs +++ b/tests/ExcelReader.Tests/Reader/ExcelDataReaderTests.cs @@ -3,7 +3,9 @@ using System.Text; using ExcelReader.Core.Reader; using ExcelReader.Core.Reader.Xlsb; +using ExcelReader.Core.Reader.Xlsx; using ExcelReader.Core.Writer.Xlsb; +using ExcelReader.Core.Writer.Xlsx; namespace ExcelReader.Tests.Reader { @@ -327,5 +329,34 @@ public async Task DataTableLoadPopulatesFromTheSheet() Assert.Equal(2.0, Convert.ToDouble(table.Rows[1]["Id"], CultureInfo.InvariantCulture)); Assert.Equal("bob", table.Rows[1]["Name"]); } + + [Fact] + public void AFailedConstructorDoesNotKeepTheReaderAlive() + { + using MemoryStream buffer = new(); + using (XlsxWorkbookWriter workbook = XlsxWorkbookWriter.Create(buffer, leaveOpen: true)) + { + XlsxSheetWriter sheet = workbook.AddSheet("s"); + using (XlsxRowWriter row = sheet.StartRow()) + { + for (int c = 0; c < 200; c++) + { + row.Write(c); + } + } + sheet.End(); + sheet.Dispose(); + workbook.End(); + } + + TrickleStream stream = new(buffer.ToArray()); + XlsxReader reader = Excel.FromXlsx(stream, leaveOpen: false, new ExcelReaderOptions { MaxCellBytes = 64 }); + Assert.Throws(() => + { + using ExcelDataReader data = new(reader); + }); + reader.Dispose(); + Assert.False(stream.CanRead); + } } } From 6256c76222e62def425243f7190e66ea6fd25967 Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Tue, 6 Oct 2026 11:17:55 -0300 Subject: [PATCH 23/28] test(core): read sheets of one reader in parallel across formats and sources Enumerators created on one thread from a single XLSX, XLSB or XLS reader (memory, stream and path sources) are drained on separate threads, sync and async, and must match a sequential read. Threads reading one sheet must get the very same shared-string instances (equal strings for XLS labels). Co-Authored-By: Claude Opus 5.5 --- .../Reader/ConcurrentSheetTests.cs | 268 ++++++++++++++++++ 1 file changed, 268 insertions(+) create mode 100644 tests/ExcelReader.Tests/Reader/ConcurrentSheetTests.cs diff --git a/tests/ExcelReader.Tests/Reader/ConcurrentSheetTests.cs b/tests/ExcelReader.Tests/Reader/ConcurrentSheetTests.cs new file mode 100644 index 00000000..3682dafa --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/ConcurrentSheetTests.cs @@ -0,0 +1,268 @@ +using System.Globalization; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Writer; +using ExcelReader.Core.Writer.Xls; +using ExcelReader.Core.Writer.Xlsb; +using ExcelReader.Core.Writer.Xlsx; + +namespace ExcelReader.Tests.Reader +{ + public sealed class ConcurrentSheetTests : IDisposable + { + public enum Format + { + Xlsx, + Xlsb, + Xls, + } + + public enum Source + { + Memory, + Stream, + Path, + } + + private const int Sheets = 6; + private const int Rows = 3000; + + private readonly List _paths = []; + + public void Dispose() + { + foreach (string path in _paths) + { + File.Delete(path); + } + } + + public static TheoryData Cases + { + get + { + TheoryData data = []; + foreach (Format format in Enum.GetValues()) + { + foreach (Source source in Enum.GetValues()) + { + data.Add(format, source); + } + } + return data; + } + } + + private static void Fill(IWorkbookWriter workbook) + where TSheet : ISheetWriter + where TRow : IRowWriter + { + for (int s = 0; s < Sheets; s++) + { + TSheet sheet = workbook.AddSheet("sheet" + s.ToString(CultureInfo.InvariantCulture)); + for (int r = 0; r < Rows; r++) + { + using TRow row = sheet.StartRow(); + row.Write("s" + s.ToString(CultureInfo.InvariantCulture) + "-name-" + (r % 200).ToString(CultureInfo.InvariantCulture)); + row.Write((s * 1_000_000) + r); + row.Write("shared-" + (r % 17).ToString(CultureInfo.InvariantCulture)); + } + sheet.End(); + sheet.Dispose(); + } + workbook.End(); + } + + private static byte[] Build(Format format) + { + using MemoryStream buffer = new(); + switch (format) + { + case Format.Xlsx: + using (XlsxWorkbookWriter xlsx = XlsxWorkbookWriter.Create(buffer, leaveOpen: true, new XlsxWriterOptions { UseSharedStrings = true })) + { + Fill(xlsx); + } + break; + case Format.Xlsb: + using (XlsbWorkbookWriter xlsb = XlsbWorkbookWriter.Create(buffer, leaveOpen: true, new XlsbWriterOptions { UseSharedStrings = true })) + { + Fill(xlsb); + } + break; + default: + using (XlsWorkbookWriter xls = XlsWorkbookWriter.Create(buffer, leaveOpen: true)) + { + Fill(xls); + } + break; + } + return buffer.ToArray(); + } + + private IExcelRowReader Open(Format format, Source source, byte[] bytes) + { + if (source == Source.Memory) + { + return Excel.Open(bytes); + } + if (source == Source.Stream) + { + return Excel.Open(new MemoryStream(bytes, writable: false), leaveOpen: false); + } + string path = Path.Combine(Path.GetTempPath(), "excelreader-concurrent-" + Guid.NewGuid().ToString("N") + "." + format.ToString()); + File.WriteAllBytes(path, bytes); + _paths.Add(path); + return Excel.Open(path); + } + + private static List Drain(IExcelRowEnumerator e) + { + List lines = []; + while (e.MoveNext()) + { + Row row = e.Current; + lines.Add(row[0].GetString() + "|" + row[1].GetString() + "|" + row[2].GetString()); + } + return lines; + } + + private static List[] ReadSequentially(IExcelRowReader reader) + { + List[] sheets = new List[reader.SheetCount]; + for (int s = 0; s < sheets.Length; s++) + { + reader.MoveToSheet(s); + using IExcelRowEnumerator e = reader.GetEnumerator(); + sheets[s] = Drain(e); + } + return sheets; + } + + [Theory] + [MemberData(nameof(Cases))] + public void Sheets_Read_In_Parallel_Match_A_Sequential_Read(Format format, Source source) + { + byte[] bytes = Build(format); + List[] expected; + using (IExcelRowReader reference = Excel.Open(bytes)) + { + expected = ReadSequentially(reference); + } + Assert.Equal(Sheets, expected.Length); + Assert.All(expected, sheet => Assert.Equal(Rows, sheet.Count)); + + using IExcelRowReader reader = Open(format, source, bytes); + IExcelRowEnumerator[] enumerators = new IExcelRowEnumerator[Sheets]; + for (int s = 0; s < Sheets; s++) + { + reader.MoveToSheet(s); + enumerators[s] = reader.GetEnumerator(); + } + + List[] actual = new List[Sheets]; + Parallel.For(0, Sheets, new ParallelOptions { MaxDegreeOfParallelism = Sheets }, s => + { + using IExcelRowEnumerator e = enumerators[s]; + actual[s] = Drain(e); + }); + + for (int s = 0; s < Sheets; s++) + { + Assert.Equal(expected[s], actual[s]); + } + } + + [Theory] + [MemberData(nameof(Cases))] + public async Task Sheets_Read_In_Parallel_Asynchronously_Match_A_Sequential_Read(Format format, Source source) + { + CancellationToken ct = TestContext.Current.CancellationToken; + byte[] bytes = Build(format); + List[] expected; + using (IExcelRowReader reference = Excel.Open(bytes)) + { + expected = ReadSequentially(reference); + } + + using IExcelRowReader reader = Open(format, source, bytes); + IExcelRowEnumerator[] enumerators = new IExcelRowEnumerator[Sheets]; + for (int s = 0; s < Sheets; s++) + { + reader.MoveToSheet(s); + enumerators[s] = reader.GetAsyncEnumerator(ct); + } + + Task>[] tasks = new Task>[Sheets]; + for (int s = 0; s < Sheets; s++) + { + IExcelRowEnumerator e = enumerators[s]; + tasks[s] = Task.Run( + async () => + { + List lines = []; + await using (e) + { + while (await e.MoveNextAsync()) + { + Row row = e.Current; + lines.Add(row[0].GetString() + "|" + row[1].GetString() + "|" + row[2].GetString()); + } + } + return lines; + }, + ct); + } + List[] actual = await Task.WhenAll(tasks); + + for (int s = 0; s < Sheets; s++) + { + Assert.Equal(expected[s], actual[s]); + } + } + + [Theory] + [MemberData(nameof(Cases))] + public void Threads_Reading_One_Sheet_Get_The_Same_String_Instances(Format format, Source source) + { + byte[] bytes = Build(format); + using IExcelRowReader reader = Open(format, source, bytes); + reader.MoveToSheet(0); + + const int Threads = 8; + IExcelRowEnumerator[] enumerators = new IExcelRowEnumerator[Threads]; + for (int t = 0; t < Threads; t++) + { + enumerators[t] = reader.GetEnumerator(); + } + + string[][] thirdColumn = new string[Threads][]; + Parallel.For(0, Threads, new ParallelOptions { MaxDegreeOfParallelism = Threads }, t => + { + using IExcelRowEnumerator e = enumerators[t]; + string[] values = new string[Rows]; + int r = 0; + while (e.MoveNext()) + { + values[r++] = e.Current[2].GetString(); + } + thirdColumn[t] = values; + }); + + for (int r = 0; r < Rows; r++) + { + Assert.Equal("shared-" + (r % 17).ToString(CultureInfo.InvariantCulture), thirdColumn[0][r]); + for (int t = 1; t < Threads; t++) + { + if (format == Format.Xls) + { + Assert.Equal(thirdColumn[0][r], thirdColumn[t][r]); + } + else + { + Assert.Same(thirdColumn[0][r], thirdColumn[t][r]); + } + } + } + } + } +} From d157b8fd2aa761ceecb12134b212e0211a622a3b Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Tue, 6 Oct 2026 12:50:51 -0300 Subject: [PATCH 24/28] fix(core): stop a disposed row enumerator from reopening its sheet Dispose left the deferred-open flag set, so a MoveNext after Dispose opened a new sheet stream and buffers with no lease. Clear the flag in Dispose and DisposeAsync, and mark the cursor at end of data once its buffers are returned so a use after dispose reports no rows instead of waiting for a source that never comes. Co-Authored-By: Claude Sonnet 5.5 --- .../Reader/Internal/BufferedStreamCursor.cs | 1 + .../Reader/PooledStreamRowEnumerator.cs | 2 ++ .../Reader/Xlsx/XlsxLifetimeTests.cs | 14 ++++++++++++++ 3 files changed, 17 insertions(+) diff --git a/src/ExcelReader.Core/Reader/Internal/BufferedStreamCursor.cs b/src/ExcelReader.Core/Reader/Internal/BufferedStreamCursor.cs index f44d7531..b6c74825 100644 --- a/src/ExcelReader.Core/Reader/Internal/BufferedStreamCursor.cs +++ b/src/ExcelReader.Core/Reader/Internal/BufferedStreamCursor.cs @@ -140,6 +140,7 @@ internal void Return() ArrayPool.Shared.Return(Buf); } Buf = []; + Eof = true; } } } diff --git a/src/ExcelReader.Core/Reader/PooledStreamRowEnumerator.cs b/src/ExcelReader.Core/Reader/PooledStreamRowEnumerator.cs index 158eff56..278e654e 100644 --- a/src/ExcelReader.Core/Reader/PooledStreamRowEnumerator.cs +++ b/src/ExcelReader.Core/Reader/PooledStreamRowEnumerator.cs @@ -79,6 +79,7 @@ public void Dispose() _source?.Dispose(); } _source = null; + _deferred = false; ReturnBuffers(); Interlocked.Exchange(ref _lease, null)?.Release(); } @@ -91,6 +92,7 @@ public async ValueTask DisposeAsync() await _source.DisposeAsync().ConfigureAwait(false); } _source = null; + _deferred = false; ReturnBuffers(); Interlocked.Exchange(ref _lease, null)?.Release(); } diff --git a/tests/ExcelReader.Tests/Reader/Xlsx/XlsxLifetimeTests.cs b/tests/ExcelReader.Tests/Reader/Xlsx/XlsxLifetimeTests.cs index 0b54470f..4009e205 100644 --- a/tests/ExcelReader.Tests/Reader/Xlsx/XlsxLifetimeTests.cs +++ b/tests/ExcelReader.Tests/Reader/Xlsx/XlsxLifetimeTests.cs @@ -114,5 +114,19 @@ public void A_Failed_GetEnumerator_Does_Not_Keep_The_Reader_Alive() reader.Dispose(); Assert.False(stream.CanRead); } + + [Fact] + public async Task A_Disposed_Deferred_Enumerator_Does_Not_Reopen_Its_Sheet() + { + TrickleStream stream = new(BuildXlsx(rows: 20)); + XlsxReader reader = Excel.FromXlsx(stream, leaveOpen: false); + XlsxReader.Enumerator e = reader.GetAsyncEnumerator(TestContext.Current.CancellationToken); + await e.DisposeAsync(); + Task move = Task.Run(async () => await e.MoveNextAsync(), TestContext.Current.CancellationToken); + Assert.Same(move, await Task.WhenAny(move, Task.Delay(TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken))); + Assert.False(await move); + reader.Dispose(); + Assert.False(stream.CanRead); + } } } From ea287905c7e077bc559eecf2e10a84cb6af4c2db Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Tue, 6 Oct 2026 12:58:02 -0300 Subject: [PATCH 25/28] test(core): cancel an XLSX shared-string load and load again Co-Authored-By: Claude Sonnet 5.5 --- .../Reader/Xlsx/XlsxLifetimeTests.cs | 50 +++++++++++++++++++ tests/ExcelReader.Tests/TestUtils.cs | 3 ++ 2 files changed, 53 insertions(+) diff --git a/tests/ExcelReader.Tests/Reader/Xlsx/XlsxLifetimeTests.cs b/tests/ExcelReader.Tests/Reader/Xlsx/XlsxLifetimeTests.cs index 4009e205..d6e0ec4b 100644 --- a/tests/ExcelReader.Tests/Reader/Xlsx/XlsxLifetimeTests.cs +++ b/tests/ExcelReader.Tests/Reader/Xlsx/XlsxLifetimeTests.cs @@ -128,5 +128,55 @@ public async Task A_Disposed_Deferred_Enumerator_Does_Not_Reopen_Its_Sheet() reader.Dispose(); Assert.False(stream.CanRead); } + + [Fact] + public async Task A_Cancelled_Shared_String_Load_Is_Undone_And_Loads_Again() + { + byte[] bytes = BuildXlsx(); + List expected; + using (XlsxReader reference = Excel.FromXlsx(bytes)) + { + using XlsxReader.Enumerator all = reference.GetEnumerator(); + expected = Drain(all); + } + + ReadOnlySpan entryName = "xl/sharedStrings.xml"u8; + int header = bytes.AsSpan().IndexOf(entryName) - 30; + long bodyStart = header + 30 + entryName.Length + BitConverter.ToUInt16(bytes, header + 28); + long bodyEnd = bodyStart + BitConverter.ToUInt32(bytes, header + 18); + + using CancellationTokenSource cts = new(); + int reads = 0; + long cancelledAt = -1; + TrickleStream stream = new(bytes); + XlsxReader reader = Excel.FromXlsx(stream, leaveOpen: false); + stream.OnRead = () => + { + if (++reads == 20) + { + cancelledAt = stream.Position; + cts.Cancel(); + } + }; + + await using (XlsxReader.Enumerator cancelled = reader.GetAsyncEnumerator(cts.Token)) + { + await Assert.ThrowsAnyAsync(async () => await cancelled.MoveNextAsync()); + } + Assert.InRange(cancelledAt, bodyStart, bodyEnd - 1); + + List actual = []; + await using (XlsxReader.Enumerator again = reader.GetAsyncEnumerator(TestContext.Current.CancellationToken)) + { + while (await again.MoveNextAsync()) + { + actual.Add(again.Current[0].GetString()); + } + } + Assert.Equal(expected, actual, StringComparer.Ordinal); + + reader.Dispose(); + Assert.False(stream.CanRead); + } } } diff --git a/tests/ExcelReader.Tests/TestUtils.cs b/tests/ExcelReader.Tests/TestUtils.cs index e46566ef..7e6653b8 100644 --- a/tests/ExcelReader.Tests/TestUtils.cs +++ b/tests/ExcelReader.Tests/TestUtils.cs @@ -158,6 +158,8 @@ internal TrickleStream(byte[] bytes) _bytes = bytes; } + internal Action? OnRead { get; set; } + public override bool CanRead => !_disposed; public override bool CanSeek => !_disposed; @@ -180,6 +182,7 @@ public override int Read(byte[] buffer, int offset, int count) public override int Read(Span buffer) { ObjectDisposedException.ThrowIf(_disposed, this); + OnRead?.Invoke(); int available = (int)Math.Max(0, Math.Min(_bytes.Length - _position, Math.Min(buffer.Length, MaxPerRead))); _bytes.AsSpan((int)_position, available).CopyTo(buffer); _position += available; From 853f1686a7d5070f1f1caf5f325c334d65b47e1f Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Tue, 6 Oct 2026 14:19:08 -0300 Subject: [PATCH 26/28] feat(core): read CSV positionally and keep the reader alive for its enumerators CsvReader over a seekable stream wraps it in a ByteSource and gives every enumerator its own positional view, so enumerators no longer share or rewind one Stream and can run concurrently. The reader holds a ReaderLifetime: Dispose closes it to new enumerators and its file or stream is released when the last enumerator is disposed. A non-seekable stream serves a single enumerator. Rewrites the thread-safety remark of IExcelRowReader accordingly. Co-Authored-By: Claude Sonnet 5.5 --- .../Reader/Csv/CsvReader.Enumerator.cs | 14 +- src/ExcelReader.Core/Reader/Csv/CsvReader.cs | 125 ++++++---- .../Reader/IExcelRowReader.cs | 10 +- .../Reader/Csv/CsvLifetimeTests.cs | 219 ++++++++++++++++++ 4 files changed, 315 insertions(+), 53 deletions(-) create mode 100644 tests/ExcelReader.Tests/Reader/Csv/CsvLifetimeTests.cs diff --git a/src/ExcelReader.Core/Reader/Csv/CsvReader.Enumerator.cs b/src/ExcelReader.Core/Reader/Csv/CsvReader.Enumerator.cs index aaba9d23..4fbc2f71 100644 --- a/src/ExcelReader.Core/Reader/Csv/CsvReader.Enumerator.cs +++ b/src/ExcelReader.Core/Reader/Csv/CsvReader.Enumerator.cs @@ -74,7 +74,12 @@ private struct FieldState } internal Enumerator(Stream stream, CsvReaderOptions options, CancellationToken ct = default) - : base(stream, options.MaxCellBytes, nameof(CsvReaderOptions.MaxCellBytes), 64 * 1024, ownsSource: false, ct) + : this(stream, options, ownsSource: false, lease: null, ct) + { + } + + internal Enumerator(Stream stream, CsvReaderOptions options, bool ownsSource, ReaderLifetime? lease, CancellationToken ct) + : base(stream, options.MaxCellBytes, nameof(CsvReaderOptions.MaxCellBytes), 64 * 1024, ownsSource, ct) { _delimiter = options.Delimiter; _quote = options.Quote; @@ -82,9 +87,15 @@ internal Enumerator(Stream stream, CsvReaderOptions options, CancellationToken c _contentCache = options.InternStrings ? new Utf8StringCache() : null; _scanner = new CsvStructuralScanner(_delimiter, _quote); _scannerSupported = CsvStructuralScanner.Supports(_delimiter, _quote); + _lease = lease; } internal Enumerator(ReadOnlyMemory content, CsvReaderOptions options, CancellationToken ct = default) + : this(content, options, lease: null, ct) + { + } + + internal Enumerator(ReadOnlyMemory content, CsvReaderOptions options, ReaderLifetime? lease, CancellationToken ct) : base(content, options.MaxCellBytes, nameof(CsvReaderOptions.MaxCellBytes), ct) { _delimiter = options.Delimiter; @@ -93,6 +104,7 @@ internal Enumerator(ReadOnlyMemory content, CsvReaderOptions options, Canc _contentCache = options.InternStrings ? new Utf8StringCache() : null; _scanner = new CsvStructuralScanner(_delimiter, _quote); _scannerSupported = CsvStructuralScanner.Supports(_delimiter, _quote); + _lease = lease; } /// diff --git a/src/ExcelReader.Core/Reader/Csv/CsvReader.cs b/src/ExcelReader.Core/Reader/Csv/CsvReader.cs index 4caf7bf5..e4cf01d7 100644 --- a/src/ExcelReader.Core/Reader/Csv/CsvReader.cs +++ b/src/ExcelReader.Core/Reader/Csv/CsvReader.cs @@ -1,5 +1,8 @@ +using System.Diagnostics.CodeAnalysis; using System.Text; using ExcelReader.Core.Parser.ParallelCsv; +using ExcelReader.Core.Reader.Internal; +using ExcelReader.Core.Reader.Sources; using ExcelReader.Core.Reader.Xls; using Microsoft.Win32.SafeHandles; @@ -12,22 +15,37 @@ namespace ExcelReader.Core.Reader.Csv /// Unlike the XLSX/XLSB/XLS readers, there are no styles or shared strings to resolve. public sealed partial class CsvReader : IExcelRowReader, IExcelRowReader { + [SuppressMessage("Usage", "CA2213:Disposable fields should be disposed", Justification = "Disposed by ReleaseResources, which the lifetime runs after the last enumerator.")] + private readonly ByteSource? _source; + private readonly long _sourceStart; + [SuppressMessage("Usage", "CA2213:Disposable fields should be disposed", Justification = "An alias of the stream the ByteSource owns; ReleaseResources disposes the source.")] + private readonly FileStream? _file; + [SuppressMessage("Usage", "CA2213:Disposable fields should be disposed", Justification = "Disposed by ReleaseResources, which the lifetime runs after the last enumerator.")] private readonly Stream? _stream; private readonly bool _leaveOpen; private readonly CsvReaderOptions _options; private readonly ReadOnlyMemory _memory; - private readonly long _startPosition = -1; - private bool _enumeratedOnce; + private int _enumeratedOnce; private SafeFileHandle? _chunkHandle; + internal ReaderLifetime Lifetime { get; } + internal CsvReader(Stream stream, bool leaveOpen, CsvReaderOptions? options = null) { ArgumentNullException.ThrowIfNull(stream); _options = options ?? CsvReaderOptions.Default; ValidateOptions(_options); - if (_options.Encoding is not null && _options.Encoding.CodePage != Encoding.UTF8.CodePage) + Lifetime = new ReaderLifetime(ReleaseResources); + if (stream.CanSeek) + { + _sourceStart = stream.Position; + _file = stream as FileStream; + _source = ByteSource.FromStream(stream, leaveOpen); + _leaveOpen = true; + } + else if (NeedsTranscoding(_options.Encoding)) { - _stream = Encoding.CreateTranscodingStream(stream, _options.Encoding, Encoding.UTF8, leaveOpen); + _stream = Encoding.CreateTranscodingStream(stream, _options.Encoding!, Encoding.UTF8, leaveOpen); _leaveOpen = false; } else @@ -35,17 +53,18 @@ internal CsvReader(Stream stream, bool leaveOpen, CsvReaderOptions? options = nu _stream = stream; _leaveOpen = leaveOpen; } - if (_stream.CanSeek) - { - _startPosition = _stream.Position; - } - _memory = default; + } + + private static bool NeedsTranscoding(Encoding? encoding) + { + return encoding is not null && encoding.CodePage != Encoding.UTF8.CodePage; } internal CsvReader(ReadOnlyMemory data, CsvReaderOptions? options = null) { _options = options ?? CsvReaderOptions.Default; ValidateOptions(_options); + Lifetime = new ReaderLifetime(ReleaseResources); _stream = null; _leaveOpen = true; _memory = Transcode(data, _options.Encoding); @@ -61,15 +80,15 @@ internal CsvReaderOptions Options internal bool TryGetChunkSource(out CsvChunkSource source) { - if (_stream is null) + if (_source is null && _stream is null) { source = new CsvChunkSource(_memory); return true; } - if (_stream is FileStream file && _startPosition >= 0) + if (_file is not null) { - SafeFileHandle handle = ChunkHandle(file); - source = new CsvChunkSource(handle, RandomAccess.GetLength(handle), _startPosition); + SafeFileHandle handle = ChunkHandle(_file); + source = new CsvChunkSource(handle, RandomAccess.GetLength(handle), _sourceStart); return true; } source = default; @@ -98,12 +117,12 @@ private SafeFileHandle ChunkHandle(FileStream file) private static ReadOnlyMemory Transcode(ReadOnlyMemory data, Encoding? encoding) { - if (encoding is null || encoding.CodePage == Encoding.UTF8.CodePage) + if (!NeedsTranscoding(encoding)) { return data; } using MemoryStream source = XlsCompoundFile.AsStream(data); - using Stream transcoding = Encoding.CreateTranscodingStream(source, encoding, Encoding.UTF8, leaveOpen: true); + using Stream transcoding = Encoding.CreateTranscodingStream(source, encoding!, Encoding.UTF8, leaveOpen: true); using MemoryStream target = new(data.Length); transcoding.CopyTo(target); return target.GetBuffer().AsMemory(0, (int)target.Length); @@ -181,14 +200,11 @@ public void MoveToSheet(int index) } /// Gets an enumerator that reads records synchronously from the start of the source. + /// The reader was disposed. + /// The source is a non-seekable stream that was already enumerated. public Enumerator GetEnumerator() { - ResetToStart(); - if (_stream is null) - { - return new Enumerator(_memory, _options); - } - return new Enumerator(_stream, _options); + return OpenSheet(); } IExcelRowEnumerator IExcelRowReader.GetEnumerator() @@ -198,15 +214,12 @@ IExcelRowEnumerator IExcelRowReader.GetEnumerator() /// Gets an enumerator that reads records asynchronously from the start of the source. /// A token observed by every MoveNextAsync call. + /// The reader was disposed. + /// The source is a non-seekable stream that was already enumerated. public Enumerator GetAsyncEnumerator(CancellationToken ct = default) { ct.ThrowIfCancellationRequested(); - ResetToStart(); - if (_stream is null) - { - return new Enumerator(_memory, _options, ct); - } - return new Enumerator(_stream, _options, ct); + return OpenSheet(ct); } IExcelRowEnumerator IExcelRowReader.GetAsyncEnumerator(CancellationToken ct) @@ -214,44 +227,62 @@ IExcelRowEnumerator IExcelRowReader.GetAsyncEnumerator(Canc return GetAsyncEnumerator(ct); } - private void ResetToStart() + internal Enumerator OpenSheet(CancellationToken ct = default) { - if (_stream is null) - { - return; - } - if (_startPosition >= 0) + Lifetime.Acquire(this); + try { - _stream.Position = _startPosition; - return; + if (_source is not null) + { + return new Enumerator(OpenSourceStream(_source), _options, ownsSource: true, Lifetime, ct); + } + if (_stream is null) + { + return new Enumerator(_memory, _options, Lifetime, ct); + } + if (Interlocked.Exchange(ref _enumeratedOnce, 1) != 0) + { + throw new InvalidOperationException( + "This CsvReader is over a non-seekable stream and can only be enumerated once."); + } + return new Enumerator(_stream, _options, ownsSource: false, Lifetime, ct); } - if (_enumeratedOnce) + catch { - throw new InvalidOperationException( - "This CsvReader is over a non-seekable stream and can only be enumerated once."); + Lifetime.Release(); + throw; } - _enumeratedOnce = true; + } + + private Stream OpenSourceStream(ByteSource source) + { + ByteSourceStream raw = new(source, _sourceStart, source.Length - _sourceStart); + return NeedsTranscoding(_options.Encoding) + ? Encoding.CreateTranscodingStream(raw, _options.Encoding!, Encoding.UTF8, leaveOpen: false) + : raw; } /// public void Dispose() { - _chunkHandle?.Dispose(); - if (!_leaveOpen && _stream is not null) - { - _stream.Dispose(); - } + Lifetime.Close(); } /// public ValueTask DisposeAsync() + { + Lifetime.Close(); + return ValueTask.CompletedTask; + } + + private void ReleaseResources() { _chunkHandle?.Dispose(); - if (_leaveOpen || _stream is null) + _source?.Dispose(); + if (!_leaveOpen) { - return ValueTask.CompletedTask; + _stream?.Dispose(); } - return _stream.DisposeAsync(); } } } diff --git a/src/ExcelReader.Core/Reader/IExcelRowReader.cs b/src/ExcelReader.Core/Reader/IExcelRowReader.cs index 71fd5e5f..4d2de72d 100644 --- a/src/ExcelReader.Core/Reader/IExcelRowReader.cs +++ b/src/ExcelReader.Core/Reader/IExcelRowReader.cs @@ -32,11 +32,11 @@ public interface IExcelRowReader /// reader () and lets callers walk every sheet without /// downcasting to the concrete XlsxReader/XlsbReader/XlsReader type. /// - /// Thread safety: no implementation is thread-safe. A reader instance carries mutable enumeration - /// state (current sheet, buffer positions, shared-string cache) with no synchronization; concurrent calls - /// from multiple threads — including two enumerators obtained from the same reader used concurrently — - /// produce undefined behavior. Use one reader instance per thread, or fully consume/dispose one enumerator - /// before starting another. + /// Thread safety: enumerators obtained from one reader may be consumed on different threads, + /// each by one thread at a time, including two enumerators over the same sheet. The reader's own + /// members are not synchronized: select sheets and create enumerators from one thread at a time. A + /// CSV reader over a non-seekable stream serves a single enumerator. Disposing the reader closes it + /// to new enumerators; its file or stream is released when the last enumerator is disposed. /// /// public interface IExcelRowReader : IExcelRowReader, IDisposable, IAsyncDisposable diff --git a/tests/ExcelReader.Tests/Reader/Csv/CsvLifetimeTests.cs b/tests/ExcelReader.Tests/Reader/Csv/CsvLifetimeTests.cs new file mode 100644 index 00000000..6d8cfe0e --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Csv/CsvLifetimeTests.cs @@ -0,0 +1,219 @@ +using System.Globalization; +using System.Text; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Csv; + +namespace ExcelReader.Tests.Reader.Csv +{ + public sealed class CsvLifetimeTests : IDisposable + { + private const int Rows = 4000; + + private readonly List _paths = []; + + public void Dispose() + { + foreach (string path in _paths) + { + File.Delete(path); + } + } + + private static byte[] BuildCsv(Encoding? encoding = null) + { + StringBuilder text = new(); + for (int r = 0; r < Rows; r++) + { + text.Append("name-").Append(r.ToString(CultureInfo.InvariantCulture)) + .Append(',').Append((r * 7).ToString(CultureInfo.InvariantCulture)) + .Append(",\"quoted, ").Append((r % 13).ToString(CultureInfo.InvariantCulture)).Append("\"\n"); + } + return (encoding ?? Encoding.UTF8).GetBytes(text.ToString()); + } + + private static List Drain(CsvReader.Enumerator e) + { + List lines = []; + while (e.MoveNext()) + { + Row row = e.Current; + lines.Add(row[0].GetString() + "|" + row[1].GetString() + "|" + row[2].GetString()); + } + return lines; + } + + private static List Expected() + { + using CsvReader reference = Excel.FromCsv(BuildCsv()); + using CsvReader.Enumerator all = reference.GetEnumerator(); + return Drain(all); + } + + [Fact] + public void Two_Enumerators_Over_One_Seekable_Stream_Do_Not_Share_A_Position() + { + List expected = Expected(); + using CsvReader reader = Excel.FromCsv(new TrickleStream(BuildCsv()), leaveOpen: false); + using CsvReader.Enumerator first = reader.GetEnumerator(); + using CsvReader.Enumerator second = reader.GetEnumerator(); + + List a = []; + List b = []; + bool moreA = true; + bool moreB = true; + while (moreA || moreB) + { + moreA = moreA && first.MoveNext(); + if (moreA) + { + a.Add(first.Current[0].GetString() + "|" + first.Current[1].GetString() + "|" + first.Current[2].GetString()); + } + moreB = moreB && second.MoveNext(); + if (moreB) + { + b.Add(second.Current[0].GetString() + "|" + second.Current[1].GetString() + "|" + second.Current[2].GetString()); + } + } + + Assert.Equal(expected, a, StringComparer.Ordinal); + Assert.Equal(expected, b, StringComparer.Ordinal); + } + + [Theory] + [InlineData("memory")] + [InlineData("stream")] + [InlineData("path")] + public void Enumerators_On_Different_Threads_Each_Read_Every_Row(string source) + { + List expected = Expected(); + byte[] bytes = BuildCsv(); + using CsvReader reader = Open(source, bytes); + + const int Threads = 6; + CsvReader.Enumerator[] enumerators = new CsvReader.Enumerator[Threads]; + for (int t = 0; t < Threads; t++) + { + enumerators[t] = reader.GetEnumerator(); + } + List[] actual = new List[Threads]; + Parallel.For(0, Threads, new ParallelOptions { MaxDegreeOfParallelism = Threads }, t => + { + using CsvReader.Enumerator e = enumerators[t]; + actual[t] = Drain(e); + }); + + Assert.All(actual, lines => Assert.Equal(expected, lines, StringComparer.Ordinal)); + } + + private CsvReader Open(string source, byte[] bytes) + { + if (string.Equals(source, "memory", StringComparison.Ordinal)) + { + return Excel.FromCsv(bytes); + } + if (string.Equals(source, "stream", StringComparison.Ordinal)) + { + return Excel.FromCsv(new TrickleStream(bytes), leaveOpen: false); + } + string path = Path.Combine(Path.GetTempPath(), "excelreader-csv-" + Guid.NewGuid().ToString("N") + ".csv"); + File.WriteAllBytes(path, bytes); + _paths.Add(path); + return Excel.FromCsvFile(path); + } + + [Fact] + public void A_Transcoded_Stream_Serves_Two_Enumerators() + { + List expected = Expected(); + using CsvReader reader = Excel.FromCsv( + new TrickleStream(BuildCsv(Encoding.Unicode)), leaveOpen: false, + new CsvReaderOptions { Encoding = Encoding.Unicode, DetectEncodingFromByteOrderMark = false }); + using CsvReader.Enumerator first = reader.GetEnumerator(); + using CsvReader.Enumerator second = reader.GetEnumerator(); + Assert.Equal(expected, Drain(first), StringComparer.Ordinal); + Assert.Equal(expected, Drain(second), StringComparer.Ordinal); + } + + [Fact] + public void A_Stream_Already_Advanced_By_The_Caller_Starts_Every_Enumerator_There() + { + List expected = Expected(); + byte[] csv = BuildCsv(); + byte[] prefix = Encoding.UTF8.GetBytes("this line is not part of the data\n"); + TrickleStream stream = new([.. prefix, .. csv]); + stream.Position = prefix.Length; + + using CsvReader reader = Excel.FromCsv(stream, leaveOpen: false); + using CsvReader.Enumerator first = reader.GetEnumerator(); + using CsvReader.Enumerator second = reader.GetEnumerator(); + Assert.Equal(expected, Drain(first), StringComparer.Ordinal); + Assert.Equal(expected, Drain(second), StringComparer.Ordinal); + } + + [Fact] + public void A_NonSeekable_Stream_Serves_Exactly_One_Enumerator() + { + using NonSeekableStream stream = new(BuildCsv()); + using CsvReader reader = Excel.FromCsv(stream, leaveOpen: true); + + int granted = 0; + int refused = 0; + CsvReader.Enumerator?[] taken = new CsvReader.Enumerator?[8]; + Parallel.For(0, taken.Length, new ParallelOptions { MaxDegreeOfParallelism = taken.Length }, t => + { + try + { + taken[t] = reader.GetEnumerator(); + Interlocked.Increment(ref granted); + } + catch (InvalidOperationException) + { + Interlocked.Increment(ref refused); + } + }); + + Assert.Equal(1, granted); + Assert.Equal(taken.Length - 1, refused); + using CsvReader.Enumerator only = taken.Single(e => e is not null)!; + Assert.Equal(Expected(), Drain(only), StringComparer.Ordinal); + } + + [Fact] + public void An_Enumerator_Outlives_Its_Reader() + { + List expected = Expected(); + TrickleStream stream = new(BuildCsv()); + CsvReader reader = Excel.FromCsv(stream, leaveOpen: false); + CsvReader.Enumerator e = reader.GetEnumerator(); + Assert.True(e.MoveNext()); + + reader.Dispose(); + Assert.True(stream.CanRead); + Assert.Equal(expected.Skip(1), Drain(e), StringComparer.Ordinal); + + e.Dispose(); + Assert.False(stream.CanRead); + } + + [Fact] + public void GetEnumerator_After_Dispose_Throws() + { + CsvReader reader = Excel.FromCsv(new TrickleStream(BuildCsv()), leaveOpen: false); + reader.Dispose(); + Assert.Throws(() => reader.GetEnumerator()); + Assert.Throws(() => reader.GetAsyncEnumerator(TestContext.Current.CancellationToken)); + } + + [Fact] + public void A_Borrowed_Stream_Is_Left_Open() + { + TrickleStream stream = new(BuildCsv()); + using (CsvReader reader = Excel.FromCsv(stream, leaveOpen: true)) + { + using CsvReader.Enumerator e = reader.GetEnumerator(); + Assert.True(e.MoveNext()); + } + Assert.True(stream.CanRead); + } + } +} From ea85fef2174fabac6f57424c4ff60c7255887462 Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Tue, 6 Oct 2026 14:34:00 -0300 Subject: [PATCH 27/28] fix(core): offer no raw chunk source for a transcoded CSV file A FileStream opened with a non-UTF-8 encoding would hand parallel chunking the raw, untranscoded file. The only caller already filters on encoding, so this was latent. Co-Authored-By: Claude Opus 5.5 --- src/ExcelReader.Core/Reader/Csv/CsvReader.cs | 2 +- .../Reader/Csv/CsvLifetimeTests.cs | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/ExcelReader.Core/Reader/Csv/CsvReader.cs b/src/ExcelReader.Core/Reader/Csv/CsvReader.cs index e4cf01d7..9dedc41a 100644 --- a/src/ExcelReader.Core/Reader/Csv/CsvReader.cs +++ b/src/ExcelReader.Core/Reader/Csv/CsvReader.cs @@ -85,7 +85,7 @@ internal bool TryGetChunkSource(out CsvChunkSource source) source = new CsvChunkSource(_memory); return true; } - if (_file is not null) + if (_file is not null && !NeedsTranscoding(_options.Encoding)) { SafeFileHandle handle = ChunkHandle(_file); source = new CsvChunkSource(handle, RandomAccess.GetLength(handle), _sourceStart); diff --git a/tests/ExcelReader.Tests/Reader/Csv/CsvLifetimeTests.cs b/tests/ExcelReader.Tests/Reader/Csv/CsvLifetimeTests.cs index 6d8cfe0e..fce632ef 100644 --- a/tests/ExcelReader.Tests/Reader/Csv/CsvLifetimeTests.cs +++ b/tests/ExcelReader.Tests/Reader/Csv/CsvLifetimeTests.cs @@ -204,6 +204,20 @@ public void GetEnumerator_After_Dispose_Throws() Assert.Throws(() => reader.GetAsyncEnumerator(TestContext.Current.CancellationToken)); } + [Fact] + public void A_Transcoded_File_Offers_No_Raw_Chunk_Source() + { + string path = Path.Combine(Path.GetTempPath(), "excelreader-csv-" + Guid.NewGuid().ToString("N") + ".csv"); + File.WriteAllBytes(path, BuildCsv(Encoding.Unicode)); + _paths.Add(path); + + using CsvReader utf16 = Excel.FromCsvFile(path, new CsvReaderOptions { Encoding = Encoding.Unicode, DetectEncodingFromByteOrderMark = false }); + Assert.False(utf16.TryGetChunkSource(out _)); + + using CsvReader utf8 = Excel.FromCsvFile(path); + Assert.True(utf8.TryGetChunkSource(out _)); + } + [Fact] public void A_Borrowed_Stream_Is_Left_Open() { From c773e7d95cfd543aae745f4abae7c8149354e52a Mon Sep 17 00:00:00 2001 From: Gabriel Matte Date: Tue, 6 Oct 2026 14:42:42 -0300 Subject: [PATCH 28/28] perf(core): read CSV sources straight into the enumerator buffer The positional stream buffered 64K of its own under a consumer that already buffers, copying every byte twice. It now takes an unbuffered mode, used for CSV when no transcoding stream sits in between. Co-Authored-By: Claude Opus 5.5 --- src/ExcelReader.Core/Reader/Csv/CsvReader.cs | 5 +++-- .../Reader/Sources/ByteSourceStream.cs | 8 +++++--- .../Reader/Sources/ByteSourceStreamTests.cs | 17 +++++++++++++++++ 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/ExcelReader.Core/Reader/Csv/CsvReader.cs b/src/ExcelReader.Core/Reader/Csv/CsvReader.cs index 9dedc41a..ae5277c4 100644 --- a/src/ExcelReader.Core/Reader/Csv/CsvReader.cs +++ b/src/ExcelReader.Core/Reader/Csv/CsvReader.cs @@ -256,8 +256,9 @@ internal Enumerator OpenSheet(CancellationToken ct = default) private Stream OpenSourceStream(ByteSource source) { - ByteSourceStream raw = new(source, _sourceStart, source.Length - _sourceStart); - return NeedsTranscoding(_options.Encoding) + bool transcode = NeedsTranscoding(_options.Encoding); + ByteSourceStream raw = new(source, _sourceStart, source.Length - _sourceStart, buffered: transcode); + return transcode ? Encoding.CreateTranscodingStream(raw, _options.Encoding!, Encoding.UTF8, leaveOpen: false) : raw; } diff --git a/src/ExcelReader.Core/Reader/Sources/ByteSourceStream.cs b/src/ExcelReader.Core/Reader/Sources/ByteSourceStream.cs index fb79e43e..6ae56e11 100644 --- a/src/ExcelReader.Core/Reader/Sources/ByteSourceStream.cs +++ b/src/ExcelReader.Core/Reader/Sources/ByteSourceStream.cs @@ -9,15 +9,17 @@ internal sealed class ByteSourceStream : Stream private readonly ByteSource _source; private readonly long _end; + private readonly bool _buffered; private long _next; private byte[]? _buffer; private int _bufferPos; private int _bufferLen; private bool _disposed; - internal ByteSourceStream(ByteSource source, long offset, long length) + internal ByteSourceStream(ByteSource source, long offset, long length, bool buffered = true) { _source = source; + _buffered = buffered; _next = offset; _end = offset + length; } @@ -55,7 +57,7 @@ public override int Read(Span destination) { return 0; } - if (destination.Length >= BufferSize) + if (!_buffered || destination.Length >= BufferSize) { int direct = _source.Read(_next, destination[..(int)Math.Min(destination.Length, remaining)]); _next += direct; @@ -87,7 +89,7 @@ public override async ValueTask ReadAsync(Memory destination, Cancell { return 0; } - if (destination.Length >= BufferSize) + if (!_buffered || destination.Length >= BufferSize) { int direct = await _source.ReadAsync( _next, destination[..(int)Math.Min(destination.Length, remaining)], cancellationToken).ConfigureAwait(false); diff --git a/tests/ExcelReader.Tests/Reader/Sources/ByteSourceStreamTests.cs b/tests/ExcelReader.Tests/Reader/Sources/ByteSourceStreamTests.cs index ed68c535..25156abd 100644 --- a/tests/ExcelReader.Tests/Reader/Sources/ByteSourceStreamTests.cs +++ b/tests/ExcelReader.Tests/Reader/Sources/ByteSourceStreamTests.cs @@ -141,5 +141,22 @@ public async Task A_Large_Async_Read_Bypasses_The_Buffer() Assert.Equal(1, source.Reads); Assert.True(big.AsSpan().SequenceEqual(bytes)); } + + [Fact] + public void An_Unbuffered_Stream_Reads_Straight_Into_The_Caller() + { + byte[] bytes = Payload(1000); + using CountingSource source = new(bytes); + using ByteSourceStream stream = new(source, 100, 800, buffered: false); + byte[] read = new byte[800]; + + Assert.Equal(10, stream.Read(read.AsSpan(0, 10))); + Assert.Equal(10, stream.Read(read.AsSpan(10, 10))); + Assert.Equal(2, source.Reads); + + stream.ReadExactly(read.AsSpan(20)); + Assert.True(read.AsSpan().SequenceEqual(bytes.AsSpan(100, 800))); + Assert.Equal(0, stream.Read(new byte[10])); + } } }