diff --git a/docs/guide/reading.md b/docs/guide/reading.md index 83ad3c87..142c5d18 100644 --- a/docs/guide/reading.md +++ b/docs/guide/reading.md @@ -107,7 +107,7 @@ using XlsxWorkbookWriter workbook = XlsxWorkbookWriter.Create(File.Create("repor workbook.WriteRecordBatch(batch); // adds the sheet, writes the header + every row, ends the workbook ``` -Supports the same seven Arrow types `ToArrowRecordBatch` produces (string, int64, double, boolean, date32, time64, timestamp); any other type throws `NotSupportedException`. Pass `writeHeader: false` to skip the header row, or a `sheetName` (XLSX/XLSB/XLS only — CSV has no sheet name) to rename it from the `"Sheet1"` default. +Supports string, large string, string view, every integer width (signed and unsigned), half float, float, double, decimal128, decimal256, boolean, date32, date64, time32, time64, timestamp and null columns; any other type (binary, nested, dictionary) throws `NotSupportedException`. Pass `writeHeader: false` to skip the header row, or a `sheetName` (XLSX/XLSB/XLS only — CSV has no sheet name) to rename it from the `"Sheet1"` default. ## Open by auto-detecting the format diff --git a/src/ExcelReader.Arrow/ArrowWriteExtensions.cs b/src/ExcelReader.Arrow/ArrowWriteExtensions.cs index b43db774..a1860064 100644 --- a/src/ExcelReader.Arrow/ArrowWriteExtensions.cs +++ b/src/ExcelReader.Arrow/ArrowWriteExtensions.cs @@ -1,3 +1,4 @@ +using System.Globalization; using Apache.Arrow; using Apache.Arrow.Types; using ExcelReader.Core.Writer; @@ -24,13 +25,14 @@ public static class ArrowWriteExtensions /// /// A freshly created, not-yet-started workbook writer. /// - /// The batch to write. Only the seven Arrow types - /// produces (string, int64, double, boolean, date32, time64, timestamp) are supported. + /// The batch to write. Supported Arrow types: string, large string, string view, every integer + /// width (signed and unsigned), half float, float, double, decimal128, decimal256, boolean, + /// date32, date64, time32, time64, timestamp and null. /// /// The sheet's name. /// Whether to write a header row of column names first. /// or is . - /// A column's Arrow type is not one of the seven supported types. + /// A column's Arrow type is not one of the supported types. public static void WriteRecordBatch(this XlsxWorkbookWriter workbook, RecordBatch batch, string sheetName = "Sheet1", bool writeHeader = true) { WriteRecordBatchCore(workbook, batch, sheetName, writeHeader); @@ -38,7 +40,7 @@ public static void WriteRecordBatch(this XlsxWorkbookWriter workbook, RecordBatc /// Same as , for an XLSB workbook. /// or is . - /// A column's Arrow type is not one of the seven supported types. + /// A column's Arrow type is not one of the supported types. public static void WriteRecordBatch(this XlsbWorkbookWriter workbook, RecordBatch batch, string sheetName = "Sheet1", bool writeHeader = true) { WriteRecordBatchCore(workbook, batch, sheetName, writeHeader); @@ -46,7 +48,7 @@ public static void WriteRecordBatch(this XlsbWorkbookWriter workbook, RecordBatc /// Same as , for a legacy XLS workbook. /// or is . - /// A column's Arrow type is not one of the seven supported types. + /// A column's Arrow type is not one of the supported types. public static void WriteRecordBatch(this XlsWorkbookWriter workbook, RecordBatch batch, string sheetName = "Sheet1", bool writeHeader = true) { WriteRecordBatchCore(workbook, batch, sheetName, writeHeader); @@ -57,7 +59,7 @@ public static void WriteRecordBatch(this XlsWorkbookWriter workbook, RecordBatch /// in a single call. /// /// or is . - /// A column's Arrow type is not one of the seven supported types. + /// A column's Arrow type is not one of the supported types. public static void WriteRecordBatch(this CsvWorkbookWriter workbook, RecordBatch batch, bool writeHeader = true) { WriteRecordBatchCore(workbook, batch, sheetName: "Sheet1", writeHeader); @@ -166,8 +168,10 @@ private static void ApplyTemporalStyles(IWorkbookWriter wo switch (fields[i].DataType.TypeId) { case ArrowTypeId.Date32: + case ArrowTypeId.Date64: sheet.SetColumnStyle(i, BuiltinDateStyleId); break; + case ArrowTypeId.Time32: case ArrowTypeId.Time64: timeStyle = timeStyle < 0 ? workbook.AddStyle(new CellStyle { NumberFormat = "hh:mm:ss" }) : timeStyle; sheet.SetColumnStyle(i, timeStyle); @@ -187,13 +191,55 @@ private static void WriteCell(IRowWriter row, IArrowArray array, ArrowTypeId typ switch (typeId) { case ArrowTypeId.String: - var strings = (StringArray)array; - if (strings.IsNull(index)) - { - row.Write((string?)null); - return; - } - row.WriteUtf8(strings.GetBytes(index)); + WriteText(row, ((StringArray)array).GetBytes(index, out bool isNull), isNull); + return; + case ArrowTypeId.LargeString: + WriteText(row, ((LargeStringArray)array).GetBytes(index, out bool isLargeNull), isLargeNull); + return; + case ArrowTypeId.StringView: + WriteText(row, ((StringViewArray)array).GetBytes(index, out bool isViewNull), isViewNull); + return; + case ArrowTypeId.Null: + row.Write((string?)null); + return; + case ArrowTypeId.Int8: + row.Write(((Int8Array)array).GetValue(index)); + return; + case ArrowTypeId.Int16: + row.Write(((Int16Array)array).GetValue(index)); + return; + case ArrowTypeId.Int32: + row.Write(((Int32Array)array).GetValue(index)); + return; + case ArrowTypeId.UInt8: + row.Write(((UInt8Array)array).GetValue(index)); + return; + case ArrowTypeId.UInt16: + row.Write(((UInt16Array)array).GetValue(index)); + return; + case ArrowTypeId.UInt32: + row.Write(((UInt32Array)array).GetValue(index)); + return; + case ArrowTypeId.UInt64: + row.Write(((UInt64Array)array).GetValue(index)); + return; + case ArrowTypeId.HalfFloat: + row.Write(ToShortestDouble(((HalfFloatArray)array).GetValue(index))); + return; + case ArrowTypeId.Float: + row.Write(ToShortestDouble(((FloatArray)array).GetValue(index))); + return; + case ArrowTypeId.Decimal128: + row.Write(((Decimal128Array)array).GetValue(index)); + return; + case ArrowTypeId.Decimal256: + row.Write(((Decimal256Array)array).GetValue(index)); + return; + case ArrowTypeId.Date64: + row.Write(((Date64Array)array).GetDateOnly(index)); + return; + case ArrowTypeId.Time32: + row.Write(((Time32Array)array).GetTime(index)); return; case ArrowTypeId.Int64: row.Write(((Int64Array)array).GetValue(index)); @@ -217,5 +263,31 @@ private static void WriteCell(IRowWriter row, IArrowArray array, ArrowTypeId typ throw new NotSupportedException($"Arrow type {typeId} is not supported for writing."); } } + + /// + /// Widens through the value's shortest decimal text, so 0.1f lands as 0.1 in every + /// format instead of the binary writers storing 0.10000000149011612. + /// + private static double? ToShortestDouble(T? value) + where T : struct, IUtf8SpanFormattable + { + if (value is not T narrow) + { + return null; + } + Span text = stackalloc byte[32]; + narrow.TryFormat(text, out int written, default, CultureInfo.InvariantCulture); + return double.Parse(text[..written], NumberStyles.Float, CultureInfo.InvariantCulture); + } + + private static void WriteText(IRowWriter row, ReadOnlySpan utf8, bool isNull) + { + if (isNull) + { + row.Write((string?)null); + return; + } + row.WriteUtf8(utf8); + } } } diff --git a/src/ExcelReader.Core/Reader/Cell.cs b/src/ExcelReader.Core/Reader/Cell.cs index c12e2d14..27ec5356 100644 --- a/src/ExcelReader.Core/Reader/Cell.cs +++ b/src/ExcelReader.Core/Reader/Cell.cs @@ -354,8 +354,14 @@ public string GetString() } if (_sharedCache is not null && !Value.IsEmpty && (uint)_sharedIndex < (uint)_sharedCache.Length) { - ref string? cached = ref _sharedCache[_sharedIndex]; - return cached ??= Encoding.UTF8.GetString(Value); + ref string? slot = ref _sharedCache[_sharedIndex]; + string? cached = Volatile.Read(ref slot); + if (cached is not null) + { + return cached; + } + string created = Encoding.UTF8.GetString(Value); + return Interlocked.CompareExchange(ref slot, created, null) ?? created; } if (_contentCache is not null && !Value.IsEmpty) { diff --git a/src/ExcelReader.Core/Reader/Csv/CsvReader.Enumerator.cs b/src/ExcelReader.Core/Reader/Csv/CsvReader.Enumerator.cs index aaba9d23..4fbc2f71 100644 --- a/src/ExcelReader.Core/Reader/Csv/CsvReader.Enumerator.cs +++ b/src/ExcelReader.Core/Reader/Csv/CsvReader.Enumerator.cs @@ -74,7 +74,12 @@ private struct FieldState } internal Enumerator(Stream stream, CsvReaderOptions options, CancellationToken ct = default) - : base(stream, options.MaxCellBytes, nameof(CsvReaderOptions.MaxCellBytes), 64 * 1024, ownsSource: false, ct) + : this(stream, options, ownsSource: false, lease: null, ct) + { + } + + internal Enumerator(Stream stream, CsvReaderOptions options, bool ownsSource, ReaderLifetime? lease, CancellationToken ct) + : base(stream, options.MaxCellBytes, nameof(CsvReaderOptions.MaxCellBytes), 64 * 1024, ownsSource, ct) { _delimiter = options.Delimiter; _quote = options.Quote; @@ -82,9 +87,15 @@ internal Enumerator(Stream stream, CsvReaderOptions options, CancellationToken c _contentCache = options.InternStrings ? new Utf8StringCache() : null; _scanner = new CsvStructuralScanner(_delimiter, _quote); _scannerSupported = CsvStructuralScanner.Supports(_delimiter, _quote); + _lease = lease; } internal Enumerator(ReadOnlyMemory content, CsvReaderOptions options, CancellationToken ct = default) + : this(content, options, lease: null, ct) + { + } + + internal Enumerator(ReadOnlyMemory content, CsvReaderOptions options, ReaderLifetime? lease, CancellationToken ct) : base(content, options.MaxCellBytes, nameof(CsvReaderOptions.MaxCellBytes), ct) { _delimiter = options.Delimiter; @@ -93,6 +104,7 @@ internal Enumerator(ReadOnlyMemory content, CsvReaderOptions options, Canc _contentCache = options.InternStrings ? new Utf8StringCache() : null; _scanner = new CsvStructuralScanner(_delimiter, _quote); _scannerSupported = CsvStructuralScanner.Supports(_delimiter, _quote); + _lease = lease; } /// diff --git a/src/ExcelReader.Core/Reader/Csv/CsvReader.cs b/src/ExcelReader.Core/Reader/Csv/CsvReader.cs index 4caf7bf5..ae5277c4 100644 --- a/src/ExcelReader.Core/Reader/Csv/CsvReader.cs +++ b/src/ExcelReader.Core/Reader/Csv/CsvReader.cs @@ -1,5 +1,8 @@ +using System.Diagnostics.CodeAnalysis; using System.Text; using ExcelReader.Core.Parser.ParallelCsv; +using ExcelReader.Core.Reader.Internal; +using ExcelReader.Core.Reader.Sources; using ExcelReader.Core.Reader.Xls; using Microsoft.Win32.SafeHandles; @@ -12,22 +15,37 @@ namespace ExcelReader.Core.Reader.Csv /// Unlike the XLSX/XLSB/XLS readers, there are no styles or shared strings to resolve. public sealed partial class CsvReader : IExcelRowReader, IExcelRowReader { + [SuppressMessage("Usage", "CA2213:Disposable fields should be disposed", Justification = "Disposed by ReleaseResources, which the lifetime runs after the last enumerator.")] + private readonly ByteSource? _source; + private readonly long _sourceStart; + [SuppressMessage("Usage", "CA2213:Disposable fields should be disposed", Justification = "An alias of the stream the ByteSource owns; ReleaseResources disposes the source.")] + private readonly FileStream? _file; + [SuppressMessage("Usage", "CA2213:Disposable fields should be disposed", Justification = "Disposed by ReleaseResources, which the lifetime runs after the last enumerator.")] private readonly Stream? _stream; private readonly bool _leaveOpen; private readonly CsvReaderOptions _options; private readonly ReadOnlyMemory _memory; - private readonly long _startPosition = -1; - private bool _enumeratedOnce; + private int _enumeratedOnce; private SafeFileHandle? _chunkHandle; + internal ReaderLifetime Lifetime { get; } + internal CsvReader(Stream stream, bool leaveOpen, CsvReaderOptions? options = null) { ArgumentNullException.ThrowIfNull(stream); _options = options ?? CsvReaderOptions.Default; ValidateOptions(_options); - if (_options.Encoding is not null && _options.Encoding.CodePage != Encoding.UTF8.CodePage) + Lifetime = new ReaderLifetime(ReleaseResources); + if (stream.CanSeek) + { + _sourceStart = stream.Position; + _file = stream as FileStream; + _source = ByteSource.FromStream(stream, leaveOpen); + _leaveOpen = true; + } + else if (NeedsTranscoding(_options.Encoding)) { - _stream = Encoding.CreateTranscodingStream(stream, _options.Encoding, Encoding.UTF8, leaveOpen); + _stream = Encoding.CreateTranscodingStream(stream, _options.Encoding!, Encoding.UTF8, leaveOpen); _leaveOpen = false; } else @@ -35,17 +53,18 @@ internal CsvReader(Stream stream, bool leaveOpen, CsvReaderOptions? options = nu _stream = stream; _leaveOpen = leaveOpen; } - if (_stream.CanSeek) - { - _startPosition = _stream.Position; - } - _memory = default; + } + + private static bool NeedsTranscoding(Encoding? encoding) + { + return encoding is not null && encoding.CodePage != Encoding.UTF8.CodePage; } internal CsvReader(ReadOnlyMemory data, CsvReaderOptions? options = null) { _options = options ?? CsvReaderOptions.Default; ValidateOptions(_options); + Lifetime = new ReaderLifetime(ReleaseResources); _stream = null; _leaveOpen = true; _memory = Transcode(data, _options.Encoding); @@ -61,15 +80,15 @@ internal CsvReaderOptions Options internal bool TryGetChunkSource(out CsvChunkSource source) { - if (_stream is null) + if (_source is null && _stream is null) { source = new CsvChunkSource(_memory); return true; } - if (_stream is FileStream file && _startPosition >= 0) + if (_file is not null && !NeedsTranscoding(_options.Encoding)) { - SafeFileHandle handle = ChunkHandle(file); - source = new CsvChunkSource(handle, RandomAccess.GetLength(handle), _startPosition); + SafeFileHandle handle = ChunkHandle(_file); + source = new CsvChunkSource(handle, RandomAccess.GetLength(handle), _sourceStart); return true; } source = default; @@ -98,12 +117,12 @@ private SafeFileHandle ChunkHandle(FileStream file) private static ReadOnlyMemory Transcode(ReadOnlyMemory data, Encoding? encoding) { - if (encoding is null || encoding.CodePage == Encoding.UTF8.CodePage) + if (!NeedsTranscoding(encoding)) { return data; } using MemoryStream source = XlsCompoundFile.AsStream(data); - using Stream transcoding = Encoding.CreateTranscodingStream(source, encoding, Encoding.UTF8, leaveOpen: true); + using Stream transcoding = Encoding.CreateTranscodingStream(source, encoding!, Encoding.UTF8, leaveOpen: true); using MemoryStream target = new(data.Length); transcoding.CopyTo(target); return target.GetBuffer().AsMemory(0, (int)target.Length); @@ -181,14 +200,11 @@ public void MoveToSheet(int index) } /// Gets an enumerator that reads records synchronously from the start of the source. + /// The reader was disposed. + /// The source is a non-seekable stream that was already enumerated. public Enumerator GetEnumerator() { - ResetToStart(); - if (_stream is null) - { - return new Enumerator(_memory, _options); - } - return new Enumerator(_stream, _options); + return OpenSheet(); } IExcelRowEnumerator IExcelRowReader.GetEnumerator() @@ -198,15 +214,12 @@ IExcelRowEnumerator IExcelRowReader.GetEnumerator() /// Gets an enumerator that reads records asynchronously from the start of the source. /// A token observed by every MoveNextAsync call. + /// The reader was disposed. + /// The source is a non-seekable stream that was already enumerated. public Enumerator GetAsyncEnumerator(CancellationToken ct = default) { ct.ThrowIfCancellationRequested(); - ResetToStart(); - if (_stream is null) - { - return new Enumerator(_memory, _options, ct); - } - return new Enumerator(_stream, _options, ct); + return OpenSheet(ct); } IExcelRowEnumerator IExcelRowReader.GetAsyncEnumerator(CancellationToken ct) @@ -214,44 +227,63 @@ IExcelRowEnumerator IExcelRowReader.GetAsyncEnumerator(Canc return GetAsyncEnumerator(ct); } - private void ResetToStart() + internal Enumerator OpenSheet(CancellationToken ct = default) { - if (_stream is null) - { - return; - } - if (_startPosition >= 0) + Lifetime.Acquire(this); + try { - _stream.Position = _startPosition; - return; + if (_source is not null) + { + return new Enumerator(OpenSourceStream(_source), _options, ownsSource: true, Lifetime, ct); + } + if (_stream is null) + { + return new Enumerator(_memory, _options, Lifetime, ct); + } + if (Interlocked.Exchange(ref _enumeratedOnce, 1) != 0) + { + throw new InvalidOperationException( + "This CsvReader is over a non-seekable stream and can only be enumerated once."); + } + return new Enumerator(_stream, _options, ownsSource: false, Lifetime, ct); } - if (_enumeratedOnce) + catch { - throw new InvalidOperationException( - "This CsvReader is over a non-seekable stream and can only be enumerated once."); + Lifetime.Release(); + throw; } - _enumeratedOnce = true; + } + + private Stream OpenSourceStream(ByteSource source) + { + bool transcode = NeedsTranscoding(_options.Encoding); + ByteSourceStream raw = new(source, _sourceStart, source.Length - _sourceStart, buffered: transcode); + return transcode + ? Encoding.CreateTranscodingStream(raw, _options.Encoding!, Encoding.UTF8, leaveOpen: false) + : raw; } /// public void Dispose() { - _chunkHandle?.Dispose(); - if (!_leaveOpen && _stream is not null) - { - _stream.Dispose(); - } + Lifetime.Close(); } /// public ValueTask DisposeAsync() + { + Lifetime.Close(); + return ValueTask.CompletedTask; + } + + private void ReleaseResources() { _chunkHandle?.Dispose(); - if (_leaveOpen || _stream is null) + _source?.Dispose(); + if (!_leaveOpen) { - return ValueTask.CompletedTask; + _stream?.Dispose(); } - return _stream.DisposeAsync(); } } } diff --git a/src/ExcelReader.Core/Reader/Csv/CsvStructuralScanner.cs b/src/ExcelReader.Core/Reader/Csv/CsvStructuralScanner.cs index 4fd09299..593a6af9 100644 --- a/src/ExcelReader.Core/Reader/Csv/CsvStructuralScanner.cs +++ b/src/ExcelReader.Core/Reader/Csv/CsvStructuralScanner.cs @@ -3,6 +3,7 @@ using System.Runtime.InteropServices; using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.X86; +using ArmAes = System.Runtime.Intrinsics.Arm.Aes; namespace ExcelReader.Core.Reader.Csv { @@ -258,14 +259,22 @@ private void Classify(ulong quotes, ulong delims, ulong crs, ulong lfs, ulong va _blocked = true; } - // ponytail: CLMUL on x86 only; ARM takes the shift ladder, add Aes.PolynomialMultiplyWideningLower if ARM throughput matters. [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static ulong PrefixXor(ulong bits) + internal static ulong PrefixXor(ulong bits) { if (Pclmulqdq.IsSupported) { return Pclmulqdq.CarrylessMultiply(Vector128.CreateScalar(bits), Vector128.AllBitsSet, 0).ToScalar(); } + if (ArmAes.IsSupported) + { + return ArmAes.PolynomialMultiplyWideningLower(Vector64.CreateScalar(bits), Vector64.AllBitsSet).ToScalar(); + } + return PrefixXorSoftware(bits); + } + + internal static ulong PrefixXorSoftware(ulong bits) + { bits ^= bits << 1; bits ^= bits << 2; bits ^= bits << 4; diff --git a/src/ExcelReader.Core/Reader/Excel.cs b/src/ExcelReader.Core/Reader/Excel.cs index fbbe7e4d..001b3f9b 100644 --- a/src/ExcelReader.Core/Reader/Excel.cs +++ b/src/ExcelReader.Core/Reader/Excel.cs @@ -1,11 +1,11 @@ using System.Buffers; using System.Diagnostics.CodeAnalysis; -using System.IO.Compression; using System.Runtime.CompilerServices; using ExcelReader.Core.Crypto; using ExcelReader.Core.Parser; using ExcelReader.Core.Reader.Csv; using ExcelReader.Core.Reader.Schema; +using ExcelReader.Core.Reader.Sources; using ExcelReader.Core.Reader.Xls; using ExcelReader.Core.Reader.Xlsb; using ExcelReader.Core.Reader.Xlsx; @@ -44,7 +44,7 @@ public static XlsxReader FromXlsx(Stream stream, bool leaveOpen = true, ExcelRea /// /// Opens an XLSX workbook directly from an in-memory buffer. Reads the ZIP - /// central directory and decompresses parts without a + /// central directory and decompresses parts without a ZipArchive /// or intermediate — every part is fully materialized up front, so the returned /// reader never suspends, even under await foreach. /// @@ -109,7 +109,7 @@ public static XlsbReader FromXlsb(Stream stream, bool leaveOpen = true, ExcelRea /// /// Opens an XLSB workbook directly from an in-memory buffer. Reads the ZIP - /// central directory and decompresses parts without a + /// central directory and decompresses parts without a ZipArchive /// or intermediate — every part is fully materialized up front, so the returned /// reader never suspends, even under await foreach. /// @@ -351,8 +351,8 @@ public static IExcelRowReader Open(Stream stream, bool leaveOpen = true, ExcelRe /// /// Opens a workbook from an in-memory buffer, auto-detecting its format (XLSX/XLSB/XLS) from its - /// signature. XLSX/XLSB route through instead of - /// a /, so the returned reader never + /// signature. XLSX/XLSB route through instead of + /// a ZipArchive/, so the returned reader never /// suspends, even under await foreach. /// /// The whole workbook file's bytes. Must outlive the returned reader. @@ -367,7 +367,7 @@ public static IExcelRowReader Open(ReadOnlyMemory data, ExcelReaderOptions ReadOnlyMemory plain = EncryptedPackageOpener.DecryptToMemory(data, effective); return OpenFromPlainMemory(plain, effective); } - ExcelFileFormat format = ClassifyMemory(data, effective, out ZipMemoryIndex? memZip); + ExcelFileFormat format = ClassifyMemory(data, effective, out ZipIndex? memZip); if (format is ExcelFileFormat.Unknown) { memZip?.Dispose(); @@ -376,15 +376,15 @@ public static IExcelRowReader Open(ReadOnlyMemory data, ExcelReaderOptions return format switch { ExcelFileFormat.Xls => new XlsReader(data, effective), - ExcelFileFormat.Xlsb => XlsbReader.CreateFromMemory(memZip!, effective), - ExcelFileFormat.Xlsx => XlsxReader.CreateFromMemory(memZip!, effective), + ExcelFileFormat.Xlsb => XlsbReader.CreateFromIndex(memZip!, effective), + ExcelFileFormat.Xlsx => XlsxReader.CreateFromIndex(memZip!, effective), _ => throw new System.Diagnostics.UnreachableException(), }; } private static IExcelRowReader OpenFromPlainMemory(ReadOnlyMemory plain, ExcelReaderOptions options) { - ExcelFileFormat format = ClassifyMemory(plain, options, out ZipMemoryIndex? memZip); + ExcelFileFormat format = ClassifyMemory(plain, options, out ZipIndex? memZip); if (format is not (ExcelFileFormat.Xlsb or ExcelFileFormat.Xlsx)) { memZip?.Dispose(); @@ -392,13 +392,13 @@ private static IExcelRowReader OpenFromPlainMemory(ReadOnlyMemory plain, E } return format switch { - ExcelFileFormat.Xlsb => XlsbReader.CreateFromMemory(memZip!, options), - ExcelFileFormat.Xlsx => XlsxReader.CreateFromMemory(memZip!, options), + ExcelFileFormat.Xlsb => XlsbReader.CreateFromIndex(memZip!, options), + ExcelFileFormat.Xlsx => XlsxReader.CreateFromIndex(memZip!, options), _ => throw new System.Diagnostics.UnreachableException(), }; } - private static ExcelFileFormat ClassifyMemory(ReadOnlyMemory data, ExcelReaderOptions options, out ZipMemoryIndex? memZip) + private static ExcelFileFormat ClassifyMemory(ReadOnlyMemory data, ExcelReaderOptions options, out ZipIndex? memZip) { memZip = null; ReadOnlySpan span = data.Span; @@ -413,8 +413,8 @@ private static ExcelFileFormat ClassifyMemory(ReadOnlyMemory data, ExcelRe ? ExcelFileFormat.EncryptedOoxml : ExcelFileFormat.Xls; } - memZip = ZipMemoryIndex.Create(data, options); - return memZip.TryGetEntry("xl/workbook.bin"u8, out _) ? ExcelFileFormat.Xlsb : ExcelFileFormat.Xlsx; + memZip = ZipIndex.Create(data, options); + return ClassifyZip(memZip); } /// @@ -629,7 +629,7 @@ private static void RequireOpenableFormat(ExcelFileFormat format) public static ExcelFileFormat DetectFileFormat(Stream stream) { ArgumentNullException.ThrowIfNull(stream); - ExcelFileFormat format = DetectSeekable(stream, out ZipArchive? zip); + ExcelFileFormat format = DetectSeekable(stream, leaveOpen: true, ExcelReaderOptions.Default, out ZipIndex? zip); zip?.Dispose(); return format; } @@ -640,7 +640,7 @@ public static ExcelFileFormat DetectFileFormat(Stream stream) /// The detected , or if the signature matches no supported format. public static ExcelFileFormat DetectFileFormat(ReadOnlyMemory data) { - var format = ClassifyMemory(data, ExcelReaderOptions.Default, out ZipMemoryIndex? memZip); + var format = ClassifyMemory(data, ExcelReaderOptions.Default, out ZipIndex? memZip); memZip?.Dispose(); return format; } @@ -654,25 +654,22 @@ public static ExcelFileFormat DetectFileFormat(ReadOnlyMemory data) public static async ValueTask DetectFileFormatAsync(Stream stream, CancellationToken ct = default) { ArgumentNullException.ThrowIfNull(stream); - (ExcelFileFormat format, ZipArchive? zip) = await DetectSeekableAsync(stream, ct).ConfigureAwait(false); - if (zip is not null) - { - await zip.DisposeAsync().ConfigureAwait(false); - } + (ExcelFileFormat format, ZipIndex? zip) = await DetectSeekableAsync(stream, leaveOpen: true, ExcelReaderOptions.Default, ct).ConfigureAwait(false); + zip?.Dispose(); return format; } private static IExcelRowReader OpenSeekable(Stream stream, bool leaveOpen, ExcelReaderOptions? options) { + ExcelReaderOptions effective = options ?? ExcelReaderOptions.Default; ExcelFileFormat format; - ZipArchive? zip = null; + ZipIndex? zip; try { - format = DetectSeekable(stream, out zip); + format = DetectSeekable(stream, leaveOpen, effective, out zip); } catch { - zip?.Dispose(); DisposeOnFailure(stream, leaveOpen); throw; } @@ -682,55 +679,37 @@ private static IExcelRowReader OpenSeekable(Stream stream, bool leaveOpen, Excel } if (format is ExcelFileFormat.EncryptedOoxml) { - ExcelReaderOptions effective = options ?? ExcelReaderOptions.Default; Stream decrypted = EncryptedPackageOpener.Decrypt(stream, leaveOpen, effective); return OpenDecryptedZip(decrypted, effective); } return format switch { ExcelFileFormat.Xls => new XlsReader(stream, leaveOpen, options), - ExcelFileFormat.Xlsb => new XlsbReader(stream, leaveOpen, zip!, options), - ExcelFileFormat.Xlsx => new XlsxReader(stream, leaveOpen, zip!, options), + ExcelFileFormat.Xlsb => XlsbReader.CreateFromIndex(zip!, effective), + ExcelFileFormat.Xlsx => XlsxReader.CreateFromIndex(zip!, effective), _ => throw new System.Diagnostics.UnreachableException(), }; } private static IExcelRowReader OpenDecryptedZip(Stream decrypted, ExcelReaderOptions options) { - ZipArchive? zip = null; - try - { - ExcelFileFormat zipFormat = ClassifyZipStream(decrypted, start: 0, out ZipArchive zipPeek); - zip = zipPeek; - return zipFormat switch - { - ExcelFileFormat.Xlsb => new XlsbReader(decrypted, leaveOpen: false, zip, options), - ExcelFileFormat.Xlsx => new XlsxReader(decrypted, leaveOpen: false, zip, options), - _ => throw new System.Diagnostics.UnreachableException(), - }; - } - catch - { - zip?.Dispose(); - decrypted.Dispose(); - throw; - } + ZipIndex zip = ZipIndex.Create(ByteSource.FromStream(decrypted, leaveOpen: false), options); + return ClassifyZip(zip) is ExcelFileFormat.Xlsb + ? XlsbReader.CreateFromIndex(zip, options) + : XlsxReader.CreateFromIndex(zip, options); } private static async ValueTask OpenSeekableAsync(Stream stream, bool leaveOpen, ExcelReaderOptions? options, CancellationToken ct) { + ExcelReaderOptions effective = options ?? ExcelReaderOptions.Default; ExcelFileFormat format; - ZipArchive? zip = null; + ZipIndex? zip; try { - (format, zip) = await DetectSeekableAsync(stream, ct).ConfigureAwait(false); + (format, zip) = await DetectSeekableAsync(stream, leaveOpen, effective, ct).ConfigureAwait(false); } catch { - if (zip is not null) - { - await zip.DisposeAsync().ConfigureAwait(false); - } await DisposeOnFailureAsync(stream, leaveOpen).ConfigureAwait(false); throw; } @@ -741,42 +720,25 @@ private static async ValueTask OpenSeekableAsync(Stream stream, } if (format is ExcelFileFormat.EncryptedOoxml) { - ExcelReaderOptions effective = options ?? ExcelReaderOptions.Default; Stream decrypted = EncryptedPackageOpener.Decrypt(stream, leaveOpen, effective); return await OpenDecryptedZipAsync(decrypted, effective, ct).ConfigureAwait(false); } return format switch { ExcelFileFormat.Xls => await XlsReader.CreateAsync(stream, leaveOpen, options, ct).ConfigureAwait(false), - ExcelFileFormat.Xlsb => await XlsbReader.CreateFromOpenZipAsync(stream, leaveOpen, zip!, options, ct).ConfigureAwait(false), - ExcelFileFormat.Xlsx => await XlsxReader.CreateFromOpenZipAsync(stream, leaveOpen, zip!, options, ct).ConfigureAwait(false), + ExcelFileFormat.Xlsb => await XlsbReader.CreateFromIndexAsync(zip!, effective, ct).ConfigureAwait(false), + ExcelFileFormat.Xlsx => await XlsxReader.CreateFromIndexAsync(zip!, effective, ct).ConfigureAwait(false), _ => throw new System.Diagnostics.UnreachableException(), }; } private static async ValueTask OpenDecryptedZipAsync(Stream decrypted, ExcelReaderOptions options, CancellationToken ct) { - ZipArchive? zip = null; - try - { - ExcelFileFormat zipFormat = ClassifyZipStream(decrypted, start: 0, out ZipArchive zipPeek); - zip = zipPeek; - return zipFormat switch - { - ExcelFileFormat.Xlsb => await XlsbReader.CreateFromOpenZipAsync(decrypted, leaveOpen: false, zip, options, ct).ConfigureAwait(false), - ExcelFileFormat.Xlsx => await XlsxReader.CreateFromOpenZipAsync(decrypted, leaveOpen: false, zip, options, ct).ConfigureAwait(false), - _ => throw new System.Diagnostics.UnreachableException(), - }; - } - catch - { - if (zip is not null) - { - await zip.DisposeAsync().ConfigureAwait(false); - } - await decrypted.DisposeAsync().ConfigureAwait(false); - throw; - } + ByteSource source = await ByteSource.FromStreamAsync(decrypted, leaveOpen: false, ct).ConfigureAwait(false); + ZipIndex zip = await ZipIndex.CreateAsync(source, options, ct).ConfigureAwait(false); + return ClassifyZip(zip) is ExcelFileFormat.Xlsb + ? await XlsbReader.CreateFromIndexAsync(zip, options, ct).ConfigureAwait(false) + : await XlsxReader.CreateFromIndexAsync(zip, options, ct).ConfigureAwait(false); } private static void DisposeOnFailure(Stream stream, bool leaveOpen) @@ -816,17 +778,13 @@ private static bool TryClassifyHeader(ReadOnlySpan sig, out ExcelFileForma return true; } - private static ExcelFileFormat ClassifyZipStream(Stream stream, long start, out ZipArchive zip) + private static ExcelFileFormat ClassifyZip(ZipIndex zip) { - var zipPeek = new ZipArchive(stream, ZipArchiveMode.Read, leaveOpen: true); - zip = zipPeek; - bool isXlsb = zipPeek.GetEntry("xl/workbook.bin") is not null; - stream.Position = start; - return isXlsb ? ExcelFileFormat.Xlsb : ExcelFileFormat.Xlsx; + return zip.TryGetEntry("xl/workbook.bin"u8, out _) ? ExcelFileFormat.Xlsb : ExcelFileFormat.Xlsx; } [SkipLocalsInit] - private static ExcelFileFormat DetectSeekable(Stream stream, out ZipArchive? zip) + private static ExcelFileFormat DetectSeekable(Stream stream, bool leaveOpen, ExcelReaderOptions options, out ZipIndex? zip) { zip = null; RequireSeekable(stream); @@ -845,12 +803,22 @@ private static ExcelFileFormat DetectSeekable(Stream stream, out ZipArchive? zip ? ExcelFileFormat.EncryptedOoxml : ExcelFileFormat.Xls; } - ExcelFileFormat zipFormat = ClassifyZipStream(stream, start, out ZipArchive zipPeek); - zip = zipPeek; - return zipFormat; + try + { + zip = ZipIndex.Create(ByteSource.FromStream(stream, leaveOpen), options); + } + finally + { + if (stream.CanSeek) + { + stream.Position = start; + } + } + return ClassifyZip(zip); } - private static async ValueTask<(ExcelFileFormat Format, ZipArchive? Zip)> DetectSeekableAsync(Stream stream, CancellationToken ct) + private static async ValueTask<(ExcelFileFormat Format, ZipIndex? Zip)> DetectSeekableAsync( + Stream stream, bool leaveOpen, ExcelReaderOptions options, CancellationToken ct) { RequireSeekable(stream); long start = stream.Position; @@ -879,8 +847,20 @@ private static ExcelFileFormat DetectSeekable(Stream stream, out ZipArchive? zip : ExcelFileFormat.Xls; return (cfbFormat, null); } - ExcelFileFormat zipFormat = ClassifyZipStream(stream, start, out ZipArchive zip); - return (zipFormat, zip); + ByteSource source = await ByteSource.FromStreamAsync(stream, leaveOpen, ct).ConfigureAwait(false); + ZipIndex zip; + try + { + zip = await ZipIndex.CreateAsync(source, options, ct).ConfigureAwait(false); + } + finally + { + if (stream.CanSeek) + { + stream.Position = start; + } + } + return (ClassifyZip(zip), zip); } private static bool TryDecryptCfbStream(Stream stream, bool leaveOpen, ExcelReaderOptions? options, out Stream decrypted) diff --git a/src/ExcelReader.Core/Reader/ExcelDataReader.cs b/src/ExcelReader.Core/Reader/ExcelDataReader.cs index 7d965b00..7ec63e4c 100644 --- a/src/ExcelReader.Core/Reader/ExcelDataReader.cs +++ b/src/ExcelReader.Core/Reader/ExcelDataReader.cs @@ -58,21 +58,29 @@ public ExcelDataReader(IExcelRowReader reader, int headerRow = 1) _sheetName = reader.SheetName; _rows = reader.GetEnumerator(); _pendingConsumed = true; - if (headerRow > 0 && SchemaInference.TrySkipToHeaderRow(_rows, headerRow, out _)) + try { - _names = ReadHeaderNames(_rows.Current); - } - else if (headerRow > 0) - { - _names = []; + if (headerRow > 0 && SchemaInference.TrySkipToHeaderRow(_rows, headerRow, out _)) + { + _names = ReadHeaderNames(_rows.Current); + } + else if (headerRow > 0) + { + _names = []; + } + else + { + _hasPendingRow = _rows.MoveNext(); + _pendingConsumed = false; + _names = _hasPendingRow ? new string?[_rows.Current.ColumnCount] : []; + } + _ordinals = BuildOrdinals(_names); } - else + catch { - _hasPendingRow = _rows.MoveNext(); - _pendingConsumed = false; - _names = _hasPendingRow ? new string?[_rows.Current.ColumnCount] : []; + _rows.Dispose(); + throw; } - _ordinals = BuildOrdinals(_names); } private static string?[] ReadHeaderNames(Row header) diff --git a/src/ExcelReader.Core/Reader/IExcelRowReader.cs b/src/ExcelReader.Core/Reader/IExcelRowReader.cs index 71fd5e5f..4d2de72d 100644 --- a/src/ExcelReader.Core/Reader/IExcelRowReader.cs +++ b/src/ExcelReader.Core/Reader/IExcelRowReader.cs @@ -32,11 +32,11 @@ public interface IExcelRowReader /// reader () and lets callers walk every sheet without /// downcasting to the concrete XlsxReader/XlsbReader/XlsReader type. /// - /// Thread safety: no implementation is thread-safe. A reader instance carries mutable enumeration - /// state (current sheet, buffer positions, shared-string cache) with no synchronization; concurrent calls - /// from multiple threads — including two enumerators obtained from the same reader used concurrently — - /// produce undefined behavior. Use one reader instance per thread, or fully consume/dispose one enumerator - /// before starting another. + /// Thread safety: enumerators obtained from one reader may be consumed on different threads, + /// each by one thread at a time, including two enumerators over the same sheet. The reader's own + /// members are not synchronized: select sheets and create enumerators from one thread at a time. A + /// CSV reader over a non-seekable stream serves a single enumerator. Disposing the reader closes it + /// to new enumerators; its file or stream is released when the last enumerator is disposed. /// /// public interface IExcelRowReader : IExcelRowReader, IDisposable, IAsyncDisposable diff --git a/src/ExcelReader.Core/Reader/Internal/BufferedStreamCursor.cs b/src/ExcelReader.Core/Reader/Internal/BufferedStreamCursor.cs index f44d7531..b6c74825 100644 --- a/src/ExcelReader.Core/Reader/Internal/BufferedStreamCursor.cs +++ b/src/ExcelReader.Core/Reader/Internal/BufferedStreamCursor.cs @@ -140,6 +140,7 @@ internal void Return() ArrayPool.Shared.Return(Buf); } Buf = []; + Eof = true; } } } diff --git a/src/ExcelReader.Core/Reader/Internal/OnceGate.cs b/src/ExcelReader.Core/Reader/Internal/OnceGate.cs new file mode 100644 index 00000000..88a18938 --- /dev/null +++ b/src/ExcelReader.Core/Reader/Internal/OnceGate.cs @@ -0,0 +1,95 @@ +using System.Runtime.ExceptionServices; + +namespace ExcelReader.Core.Reader.Internal +{ + /// + /// Runs a load once for however many callers ask. A failure is kept and rethrown; a cancelled + /// load is undone so the next caller starts clean. + /// + internal sealed class OnceGate : IDisposable + { + private readonly SemaphoreSlim _gate = new(1, 1); + private volatile bool _done; + private ExceptionDispatchInfo? _failure; + + internal bool IsDone => _done; + + internal void Run(TState state, Action load, Action reset) + { + if (_done) + { + return; + } + _gate.Wait(); + try + { + if (_done) + { + return; + } + _failure?.Throw(); + try + { + load(state); + } + catch (OperationCanceledException) + { + reset(state); + throw; + } + catch (Exception ex) + { + _failure = ExceptionDispatchInfo.Capture(ex); + throw; + } + _done = true; + } + finally + { + _gate.Release(); + } + } + + internal async ValueTask RunAsync( + TState state, Func load, Action reset, CancellationToken ct) + { + if (_done) + { + return; + } + await _gate.WaitAsync(ct).ConfigureAwait(false); + try + { + if (_done) + { + return; + } + _failure?.Throw(); + try + { + await load(state, ct).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + reset(state); + throw; + } + catch (Exception ex) + { + _failure = ExceptionDispatchInfo.Capture(ex); + throw; + } + _done = true; + } + finally + { + _gate.Release(); + } + } + + public void Dispose() + { + _gate.Dispose(); + } + } +} diff --git a/src/ExcelReader.Core/Reader/Internal/ReaderLifetime.cs b/src/ExcelReader.Core/Reader/Internal/ReaderLifetime.cs new file mode 100644 index 00000000..4ea572dd --- /dev/null +++ b/src/ExcelReader.Core/Reader/Internal/ReaderLifetime.cs @@ -0,0 +1,41 @@ +namespace ExcelReader.Core.Reader.Internal +{ + /// + /// Keeps a reader's resources alive while any enumerator still uses them: the reader holds one + /// reference, each enumerator another, and the release runs when the last one goes. + /// + internal sealed class ReaderLifetime(Action release) + { + private int _count = 1; + private int _closed; + + internal void Acquire(object owner) + { + while (true) + { + int current = Volatile.Read(ref _count); + ObjectDisposedException.ThrowIf(current == 0 || Volatile.Read(ref _closed) != 0, owner); + if (Interlocked.CompareExchange(ref _count, current + 1, current) == current) + { + return; + } + } + } + + internal void Release() + { + if (Interlocked.Decrement(ref _count) == 0) + { + release(); + } + } + + internal void Close() + { + if (Interlocked.Exchange(ref _closed, 1) == 0) + { + Release(); + } + } + } +} diff --git a/src/ExcelReader.Core/Reader/Internal/WorkbookLookups.cs b/src/ExcelReader.Core/Reader/Internal/WorkbookLookups.cs index e414e0c5..186910ae 100644 --- a/src/ExcelReader.Core/Reader/Internal/WorkbookLookups.cs +++ b/src/ExcelReader.Core/Reader/Internal/WorkbookLookups.cs @@ -1,4 +1,3 @@ -using System.IO.Compression; using System.Runtime.CompilerServices; using System.Text; using ExcelReader.Core.Reader.Zip; @@ -66,14 +65,8 @@ internal static (int Start, int Length, int ValidIndex) SharedAt(int[] sharedOff return (sharedOffsets[index], sharedOffsets[index + 1] - sharedOffsets[index], index); } - internal static ZipArchiveEntry GetWorksheetEntry(ZipArchive zip, string path) - { - return zip.GetEntry(path) - ?? throw new InvalidDataException($"Worksheet part not found: {path}"); - } - [SkipLocalsInit] - internal static ZipEntryRef GetWorksheetEntry(ZipMemoryIndex memZip, string path) + internal static ZipEntryRef GetWorksheetEntry(ZipIndex memZip, string path) { Span stackBuffer = stackalloc byte[256]; ReadOnlySpan utf8Path = Encoding.UTF8.GetByteCount(path) <= stackBuffer.Length @@ -84,21 +77,6 @@ internal static ZipEntryRef GetWorksheetEntry(ZipMemoryIndex memZip, string path : throw new InvalidDataException($"Worksheet part not found: {path}"); } - internal static LimitedReadStream OpenEntryStream( - ZipArchiveEntry entry, DecompressedByteCounter counter, ExcelReaderOptions options, - string entryLimitName = "", long entryLimit = 0) - { - return Wrap(entry.Open(), counter, options, entryLimitName, entryLimit, entry.Length); - } - - internal static async ValueTask OpenEntryStreamAsync( - ZipArchiveEntry entry, DecompressedByteCounter counter, ExcelReaderOptions options, - CancellationToken ct, string entryLimitName = "", long entryLimit = 0) - { - Stream opened = await entry.OpenAsync(ct).ConfigureAwait(false); - return Wrap(opened, counter, options, entryLimitName, entryLimit, entry.Length); - } - private const long PrefetchMinUncompressedSize = 256 * 1024; internal static LimitedReadStream Wrap( diff --git a/src/ExcelReader.Core/Reader/PooledStreamRowEnumerator.cs b/src/ExcelReader.Core/Reader/PooledStreamRowEnumerator.cs index 2cac3766..278e654e 100644 --- a/src/ExcelReader.Core/Reader/PooledStreamRowEnumerator.cs +++ b/src/ExcelReader.Core/Reader/PooledStreamRowEnumerator.cs @@ -14,6 +14,7 @@ public abstract class PooledStreamRowEnumerator private protected readonly BufferedStreamCursor _io; private protected readonly CellAccumulator _acc; private readonly bool _ownsSource; + private protected ReaderLifetime? _lease; private bool _deferred; private protected byte[] _buf => _io.Buf; private protected int _pos { get => _io.Pos; set => _io.Pos = value; } @@ -78,7 +79,9 @@ public void Dispose() _source?.Dispose(); } _source = null; + _deferred = false; ReturnBuffers(); + Interlocked.Exchange(ref _lease, null)?.Release(); } /// Asynchronous counterpart to . @@ -89,7 +92,9 @@ public async ValueTask DisposeAsync() await _source.DisposeAsync().ConfigureAwait(false); } _source = null; + _deferred = false; ReturnBuffers(); + Interlocked.Exchange(ref _lease, null)?.Release(); } private protected void Fill() diff --git a/src/ExcelReader.Core/Reader/Sources/ByteSource.cs b/src/ExcelReader.Core/Reader/Sources/ByteSource.cs new file mode 100644 index 00000000..1801ead6 --- /dev/null +++ b/src/ExcelReader.Core/Reader/Sources/ByteSource.cs @@ -0,0 +1,137 @@ +using Microsoft.Win32.SafeHandles; + +namespace ExcelReader.Core.Reader.Sources +{ + /// + /// Random-access bytes of one workbook file. Every read names its own offset, so readers of the + /// same source share no cursor. + /// + internal abstract class ByteSource : IDisposable + { + internal abstract long Length { get; } + + internal abstract int Read(long offset, Span destination); + + internal abstract ValueTask ReadAsync(long offset, Memory destination, CancellationToken ct); + + internal virtual bool TryGetMemory(out ReadOnlyMemory memory) + { + memory = default; + return false; + } + + internal void ReadExactly(long offset, Span destination) + { + int filled = 0; + while (filled < destination.Length) + { + int read = Read(offset + filled, destination[filled..]); + if (read <= 0) + { + throw new EndOfStreamException(); + } + filled += read; + } + } + + internal async ValueTask ReadExactlyAsync(long offset, Memory destination, CancellationToken ct) + { + int filled = 0; + while (filled < destination.Length) + { + int read = await ReadAsync(offset + filled, destination[filled..], ct).ConfigureAwait(false); + if (read <= 0) + { + throw new EndOfStreamException(); + } + filled += read; + } + } + + public abstract void Dispose(); + + internal static ByteSource FromMemory(ReadOnlyMemory data) + { + return new MemoryByteSource(data, owner: null); + } + + internal static ByteSource OpenFile(string path, bool asynchronous) + { + SafeFileHandle handle = File.OpenHandle(path, FileMode.Open, FileAccess.Read, FileShare.Read, + asynchronous ? FileOptions.Asynchronous : FileOptions.None); + try + { + return new FileByteSource(handle, RandomAccess.GetLength(handle), owner: handle); + } + catch + { + handle.Dispose(); + throw; + } + } + + internal static ByteSource FromStream(Stream stream, bool leaveOpen) + { + if (TryWrapSeekable(stream, leaveOpen, out ByteSource? source)) + { + return source; + } + try + { + using MemoryStream copy = new(); + stream.CopyTo(copy); + return new MemoryByteSource(copy.GetBuffer().AsMemory(0, (int)copy.Length), owner: null); + } + finally + { + if (!leaveOpen) + { + stream.Dispose(); + } + } + } + + internal static async ValueTask FromStreamAsync(Stream stream, bool leaveOpen, CancellationToken ct) + { + if (TryWrapSeekable(stream, leaveOpen, out ByteSource? source)) + { + return source; + } + try + { + using MemoryStream copy = new(); + await stream.CopyToAsync(copy, ct).ConfigureAwait(false); + return new MemoryByteSource(copy.GetBuffer().AsMemory(0, (int)copy.Length), owner: null); + } + finally + { + if (!leaveOpen) + { + await stream.DisposeAsync().ConfigureAwait(false); + } + } + } + + private static bool TryWrapSeekable(Stream stream, bool leaveOpen, [System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out ByteSource? source) + { + IDisposable? owner = leaveOpen ? null : stream; + if (stream.GetType() == typeof(MemoryStream) && ((MemoryStream)stream).TryGetBuffer(out ArraySegment segment)) + { + source = new MemoryByteSource(segment, owner); + return true; + } + if (stream.GetType() == typeof(FileStream) && stream.CanSeek) + { + source = new FileByteSource(((FileStream)stream).SafeFileHandle, stream.Length, owner); + return true; + } + if (stream.CanSeek) + { + source = new StreamByteSource(stream, leaveOpen); + return true; + } + source = null; + return false; + } + } +} diff --git a/src/ExcelReader.Core/Reader/Sources/ByteSourceStream.cs b/src/ExcelReader.Core/Reader/Sources/ByteSourceStream.cs new file mode 100644 index 00000000..6ae56e11 --- /dev/null +++ b/src/ExcelReader.Core/Reader/Sources/ByteSourceStream.cs @@ -0,0 +1,155 @@ +using System.Buffers; + +namespace ExcelReader.Core.Reader.Sources +{ + /// Forward-only view of one byte range of a , with its own offset. + internal sealed class ByteSourceStream : Stream + { + private const int BufferSize = 64 * 1024; + + private readonly ByteSource _source; + private readonly long _end; + private readonly bool _buffered; + private long _next; + private byte[]? _buffer; + private int _bufferPos; + private int _bufferLen; + private bool _disposed; + + internal ByteSourceStream(ByteSource source, long offset, long length, bool buffered = true) + { + _source = source; + _buffered = buffered; + _next = offset; + _end = offset + length; + } + + public override bool CanRead => true; + + public override bool CanSeek => false; + + public override bool CanWrite => false; + + public override long Length => throw new NotSupportedException(); + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + return Read(buffer.AsSpan(offset, count)); + } + + public override int Read(Span destination) + { + ObjectDisposedException.ThrowIf(_disposed, this); + if (destination.IsEmpty) + { + return 0; + } + if (_bufferPos == _bufferLen) + { + long remaining = _end - _next; + if (remaining <= 0) + { + return 0; + } + if (!_buffered || destination.Length >= BufferSize) + { + int direct = _source.Read(_next, destination[..(int)Math.Min(destination.Length, remaining)]); + _next += direct; + return direct; + } + _buffer ??= ArrayPool.Shared.Rent(BufferSize); + _bufferLen = _source.Read(_next, _buffer.AsSpan(0, (int)Math.Min(BufferSize, remaining))); + _bufferPos = 0; + _next += _bufferLen; + if (_bufferLen == 0) + { + return 0; + } + } + return TakeBuffered(destination); + } + + public override async ValueTask ReadAsync(Memory destination, CancellationToken cancellationToken = default) + { + ObjectDisposedException.ThrowIf(_disposed, this); + if (destination.IsEmpty) + { + return 0; + } + if (_bufferPos == _bufferLen) + { + long remaining = _end - _next; + if (remaining <= 0) + { + return 0; + } + if (!_buffered || destination.Length >= BufferSize) + { + int direct = await _source.ReadAsync( + _next, destination[..(int)Math.Min(destination.Length, remaining)], cancellationToken).ConfigureAwait(false); + _next += direct; + return direct; + } + _buffer ??= ArrayPool.Shared.Rent(BufferSize); + _bufferLen = await _source.ReadAsync( + _next, _buffer.AsMemory(0, (int)Math.Min(BufferSize, remaining)), cancellationToken).ConfigureAwait(false); + _bufferPos = 0; + _next += _bufferLen; + if (_bufferLen == 0) + { + return 0; + } + } + return TakeBuffered(destination.Span); + } + + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + return ReadAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask(); + } + + private int TakeBuffered(Span destination) + { + int count = Math.Min(destination.Length, _bufferLen - _bufferPos); + _buffer.AsSpan(_bufferPos, count).CopyTo(destination); + _bufferPos += count; + return count; + } + + public override void Flush() + { + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotSupportedException(); + } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotSupportedException(); + } + + protected override void Dispose(bool disposing) + { + _disposed = true; + byte[]? buffer = Interlocked.Exchange(ref _buffer, null); + if (buffer is not null) + { + ArrayPool.Shared.Return(buffer); + } + base.Dispose(disposing); + } + } +} diff --git a/src/ExcelReader.Core/Reader/Sources/FileByteSource.cs b/src/ExcelReader.Core/Reader/Sources/FileByteSource.cs new file mode 100644 index 00000000..ebd9f584 --- /dev/null +++ b/src/ExcelReader.Core/Reader/Sources/FileByteSource.cs @@ -0,0 +1,36 @@ +using Microsoft.Win32.SafeHandles; + +namespace ExcelReader.Core.Reader.Sources +{ + internal sealed class FileByteSource : ByteSource + { + private readonly SafeFileHandle _handle; + private readonly IDisposable? _owner; + + internal FileByteSource(SafeFileHandle handle, long length, IDisposable? owner) + { + _handle = handle; + Length = length; + _owner = owner; + } + + internal override long Length { get; } + + internal override int Read(long offset, Span destination) + { + ArgumentOutOfRangeException.ThrowIfNegative(offset); + return offset >= Length ? 0 : RandomAccess.Read(_handle, destination, offset); + } + + internal override ValueTask ReadAsync(long offset, Memory destination, CancellationToken ct) + { + ArgumentOutOfRangeException.ThrowIfNegative(offset); + return offset >= Length ? new ValueTask(0) : RandomAccess.ReadAsync(_handle, destination, offset, ct); + } + + public override void Dispose() + { + _owner?.Dispose(); + } + } +} diff --git a/src/ExcelReader.Core/Reader/Sources/MemoryByteSource.cs b/src/ExcelReader.Core/Reader/Sources/MemoryByteSource.cs new file mode 100644 index 00000000..68a66d85 --- /dev/null +++ b/src/ExcelReader.Core/Reader/Sources/MemoryByteSource.cs @@ -0,0 +1,45 @@ +namespace ExcelReader.Core.Reader.Sources +{ + internal sealed class MemoryByteSource : ByteSource + { + private readonly ReadOnlyMemory _memory; + private readonly IDisposable? _owner; + + internal MemoryByteSource(ReadOnlyMemory memory, IDisposable? owner) + { + _memory = memory; + _owner = owner; + } + + internal override long Length => _memory.Length; + + internal override int Read(long offset, Span destination) + { + ArgumentOutOfRangeException.ThrowIfNegative(offset); + if (offset >= _memory.Length) + { + return 0; + } + int count = (int)Math.Min(destination.Length, _memory.Length - offset); + _memory.Span.Slice((int)offset, count).CopyTo(destination); + return count; + } + + internal override ValueTask ReadAsync(long offset, Memory destination, CancellationToken ct) + { + ArgumentOutOfRangeException.ThrowIfNegative(offset); + return new ValueTask(Read(offset, destination.Span)); + } + + internal override bool TryGetMemory(out ReadOnlyMemory memory) + { + memory = _memory; + return true; + } + + public override void Dispose() + { + _owner?.Dispose(); + } + } +} diff --git a/src/ExcelReader.Core/Reader/Sources/StreamByteSource.cs b/src/ExcelReader.Core/Reader/Sources/StreamByteSource.cs new file mode 100644 index 00000000..d8c6bc61 --- /dev/null +++ b/src/ExcelReader.Core/Reader/Sources/StreamByteSource.cs @@ -0,0 +1,69 @@ +namespace ExcelReader.Core.Reader.Sources +{ + /// + /// A seekable stream has one cursor, so each read takes the gate for its seek and read only; + /// decompression and parsing happen outside it. + /// + internal sealed class StreamByteSource : ByteSource + { + private readonly Stream _stream; + private readonly bool _leaveOpen; + private readonly SemaphoreSlim _gate = new(1, 1); + + internal StreamByteSource(Stream stream, bool leaveOpen) + { + _stream = stream; + _leaveOpen = leaveOpen; + Length = stream.Length; + } + + internal override long Length { get; } + + internal override int Read(long offset, Span destination) + { + ArgumentOutOfRangeException.ThrowIfNegative(offset); + if (offset >= Length) + { + return 0; + } + _gate.Wait(); + try + { + _stream.Position = offset; + return _stream.Read(destination); + } + finally + { + _gate.Release(); + } + } + + internal override async ValueTask ReadAsync(long offset, Memory destination, CancellationToken ct) + { + ArgumentOutOfRangeException.ThrowIfNegative(offset); + if (offset >= Length) + { + return 0; + } + await _gate.WaitAsync(ct).ConfigureAwait(false); + try + { + _stream.Position = offset; + return await _stream.ReadAsync(destination, ct).ConfigureAwait(false); + } + finally + { + _gate.Release(); + } + } + + public override void Dispose() + { + if (!_leaveOpen) + { + _stream.Dispose(); + } + _gate.Dispose(); + } + } +} diff --git a/src/ExcelReader.Core/Reader/Xls/WorkbookStream.cs b/src/ExcelReader.Core/Reader/Xls/WorkbookStream.cs index 42c430b5..78ed0d7a 100644 --- a/src/ExcelReader.Core/Reader/Xls/WorkbookStream.cs +++ b/src/ExcelReader.Core/Reader/Xls/WorkbookStream.cs @@ -1,6 +1,7 @@ using System.Buffers; using System.Diagnostics.CodeAnalysis; using System.Runtime.InteropServices; +using ExcelReader.Core.Reader.Sources; namespace ExcelReader.Core.Reader.Xls { @@ -9,8 +10,7 @@ internal sealed class WorkbookStream : IDisposable { private const int HeaderSize = 512; - private readonly Stream? _source; - private readonly bool _ownsSource; + private readonly ByteSource? _source; private readonly int[] _chain; private readonly int _chainLength; private bool _chainReturned; @@ -29,10 +29,9 @@ internal enum SourceKind internal SourceKind Kind { get; } - private WorkbookStream(Stream? source, bool ownsSource, int[] chain, int chainLength, ReadOnlyMemory memory, int sectorSize, long length, SourceKind kind) + private WorkbookStream(ByteSource? source, int[] chain, int chainLength, ReadOnlyMemory memory, int sectorSize, long length, SourceKind kind) { _source = source; - _ownsSource = ownsSource; _chain = chain; _chainLength = chainLength; SectorSize = sectorSize; @@ -54,19 +53,19 @@ private WorkbookStream(Stream? source, bool ownsSource, int[] chain, int chainLe _fileLength = Buffer.Length; } - internal static WorkbookStream Streamed(Stream source, bool ownsSource, int[] chain, int chainLength, int sectorSize, long length) + internal static WorkbookStream Streamed(ByteSource source, int[] chain, int chainLength, int sectorSize, long length) { - return new WorkbookStream(source, ownsSource, chain, chainLength, default, sectorSize, length, SourceKind.Streamed); + return new WorkbookStream(source, chain, chainLength, default, sectorSize, length, SourceKind.Streamed); } internal static WorkbookStream InMemory(ReadOnlyMemory data) { - return new WorkbookStream(null, ownsSource: false, [], 0, data, sectorSize: 1, data.Length, SourceKind.Contiguous); + return new WorkbookStream(null, [], 0, data, sectorSize: 1, data.Length, SourceKind.Contiguous); } internal static WorkbookStream Chained(ReadOnlyMemory data, int[] chain, int chainLength, int sectorSize, long length) { - return new WorkbookStream(null, ownsSource: false, chain, chainLength, data, sectorSize, length, SourceKind.Chained); + return new WorkbookStream(null, chain, chainLength, data, sectorSize, length, SourceKind.Chained); } internal BiffCursor OpenCursor() @@ -156,17 +155,13 @@ internal int LoadSectors(int chainIndex, Span dest) count++; } long offset = HeaderSize + ((long)_chain[chainIndex] * SectorSize); - _source!.Seek(offset, SeekOrigin.Begin); - _source.ReadExactly(dest[..(count * SectorSize)]); + _source!.ReadExactly(offset, dest[..(count * SectorSize)]); return count; } public void Dispose() { - if (_ownsSource) - { - _source?.Dispose(); - } + _source?.Dispose(); if (_chainLength > 0 && !_chainReturned) { _chainReturned = true; diff --git a/src/ExcelReader.Core/Reader/Xls/XlsCompoundFile.cs b/src/ExcelReader.Core/Reader/Xls/XlsCompoundFile.cs index 670170f1..c7771f65 100644 --- a/src/ExcelReader.Core/Reader/Xls/XlsCompoundFile.cs +++ b/src/ExcelReader.Core/Reader/Xls/XlsCompoundFile.cs @@ -2,6 +2,7 @@ using System.Runtime.InteropServices; using ExcelReader.Core.Crypto; using ExcelReader.Core.Reader.Internal; +using ExcelReader.Core.Reader.Sources; namespace ExcelReader.Core.Reader.Xls { @@ -138,7 +139,7 @@ private static WorkbookStream BuildWorkbook(Stream source, bool ownsSource, Exce { return WorkbookStream.Chained(memory, chain, chainCount, cfb.SectorSize, workbook.Size); } - return WorkbookStream.Streamed(source, ownsSource, chain, chainCount, cfb.SectorSize, workbook.Size); + return WorkbookStream.Streamed(ByteSource.FromStream(source, leaveOpen: !ownsSource), chain, chainCount, cfb.SectorSize, workbook.Size); } finally { diff --git a/src/ExcelReader.Core/Reader/Xls/XlsReader.Enumerator.cs b/src/ExcelReader.Core/Reader/Xls/XlsReader.Enumerator.cs index deee8b98..2f8cabcd 100644 --- a/src/ExcelReader.Core/Reader/Xls/XlsReader.Enumerator.cs +++ b/src/ExcelReader.Core/Reader/Xls/XlsReader.Enumerator.cs @@ -17,8 +17,12 @@ public sealed class Enumerator : IExcelRowEnumerator private readonly BiffCursor _cursor; private readonly CellAccumulator _acc; private readonly Utf8StringCache? _contentCache; + private readonly Queue _declaredRows = new(); private bool _ended; + private bool _emptyRow; private int _row; + private int _rowFloor = -1; + private ReaderLifetime? _lease; internal Enumerator(XlsReader reader, int sheetOffset, CancellationToken ct = default) { @@ -28,6 +32,7 @@ internal Enumerator(XlsReader reader, int sheetOffset, CancellationToken ct = de _row = -1; _acc = new CellAccumulator(reader._options.MaxCellBytes, nameof(ExcelReaderOptions.MaxCellBytes)); _contentCache = reader._options.InternStrings ? new Utf8StringCache() : null; + _lease = reader.Lifetime; } /// @@ -72,7 +77,8 @@ private bool MoveNextCore() return true; } } - return false; + ResetRow(); + return _declaredRows.TryDequeue(out _); } private bool ReadRecord(BiffCursor cursor, long recordStart, int id, ReadOnlySpan data) @@ -96,6 +102,9 @@ private bool ReadRecord(BiffCursor cursor, long recordStart, int id, ReadOnlySpa } switch (id) { + case Rec.Row: + DeclareRow(ReadU16(data, 0)); + return true; case Rec.Label: if (!AdvanceRow(cursor, recordStart, ReadU16(data, 0))) { return false; } ParseLabel(data); @@ -144,11 +153,32 @@ private bool ReadRecord(BiffCursor cursor, long recordStart, int id, ReadOnlySpa } } + /// Queues a ROW record's row so it is yielded, empty, if no cell record claims it. + private void DeclareRow(int row) + { + if (row > _rowFloor) + { + _rowFloor = row; + _declaredRows.Enqueue(row); + } + } + private bool AdvanceRow(BiffCursor cursor, long recordStart, int row) { if (_row < 0) { + if (_declaredRows.TryPeek(out int declared) && declared <= row) + { + _declaredRows.Dequeue(); + if (declared < row) + { + cursor.Position = recordStart; + _emptyRow = true; + return false; + } + } _row = row; + _rowFloor = Math.Max(_rowFloor, row); return true; } if (row != _row) @@ -162,13 +192,14 @@ private bool AdvanceRow(BiffCursor cursor, long recordStart, int row) private bool FinishRow() { _acc.SortByColumn(); - return _acc.Count > 0; + return _row >= 0 || _emptyRow; } private void ResetRow() { _acc.Reset(); _row = -1; + _emptyRow = false; } private void ParseLabel(ReadOnlySpan data) @@ -303,6 +334,7 @@ private void ReturnBuffers() { _cursor.Dispose(); _acc.Return(); + Interlocked.Exchange(ref _lease, null)?.Release(); } } } diff --git a/src/ExcelReader.Core/Reader/Xls/XlsReader.cs b/src/ExcelReader.Core/Reader/Xls/XlsReader.cs index 8ae7e95d..89ef3229 100644 --- a/src/ExcelReader.Core/Reader/Xls/XlsReader.cs +++ b/src/ExcelReader.Core/Reader/Xls/XlsReader.cs @@ -18,6 +18,8 @@ public sealed partial class XlsReader : IExcelRowReader, IExcelRowReader data, ExcelReaderOptions? options = null private XlsReader(WorkbookStream workbook, ExcelReaderOptions? options = null) { + Lifetime = new ReaderLifetime(ReleaseResources); _workbook = workbook; _options = options ?? ExcelReaderOptions.Default; using (BiffCursor cursor = workbook.OpenCursor()) @@ -84,7 +87,13 @@ public ExcelSheetVisibility SheetVisibilityAt(int index) internal ReadOnlySpan SharedSpan => _sharedFlat; - internal string?[] SharedStringCache => _sharedStringCache ??= WorkbookLookups.CreateSharedStringCache(_sharedOffsets); + internal string?[] SharedStringCache => Volatile.Read(ref _sharedStringCache) ?? CreateSharedStringCache(); + + private string?[] CreateSharedStringCache() + { + string?[] created = WorkbookLookups.CreateSharedStringCache(_sharedOffsets); + return Interlocked.CompareExchange(ref _sharedStringCache, created, null) ?? created; + } internal bool IsDateStyle(int style) { @@ -117,7 +126,16 @@ public void MoveToSheet(int index) /// public Enumerator GetEnumerator() { - return new Enumerator(this, _sheets[_current].Offset); + Lifetime.Acquire(this); + try + { + return new Enumerator(this, _sheets[_current].Offset); + } + catch + { + Lifetime.Release(); + throw; + } } IExcelRowEnumerator IExcelRowReader.GetEnumerator() @@ -130,7 +148,16 @@ IExcelRowEnumerator IExcelRowReader.GetEnumerator() public Enumerator GetAsyncEnumerator(CancellationToken ct = default) { ct.ThrowIfCancellationRequested(); - return new Enumerator(this, _sheets[_current].Offset, ct); + Lifetime.Acquire(this); + try + { + return new Enumerator(this, _sheets[_current].Offset, ct); + } + catch + { + Lifetime.Release(); + throw; + } } IExcelRowEnumerator IExcelRowReader.GetAsyncEnumerator(CancellationToken ct) @@ -141,19 +168,24 @@ IExcelRowEnumerator IExcelRowReader.GetAsyncEnumerator(Canc /// public void Dispose() { - _workbook.Dispose(); - _sharedFlat = []; - _sharedOffsets = [0]; - _sharedStringCache = null; + Lifetime.Close(); } /// public ValueTask DisposeAsync() { - Dispose(); + Lifetime.Close(); return ValueTask.CompletedTask; } + private void ReleaseResources() + { + _workbook.Dispose(); + _sharedFlat = []; + _sharedOffsets = [0]; + _sharedStringCache = null; + } + private static void ParseWorkbookGlobals( BiffCursor cursor, ExcelReaderOptions options, @@ -428,6 +460,7 @@ private static class Rec internal const int MulRk = 0x00BD; internal const int BoolErr = 0x0205; internal const int Formula = 0x0006; + internal const int Row = 0x0208; internal const int Blank = 0x0201; internal const int MulBlank = 0x00BE; internal const int StringRec = 0x0207; diff --git a/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Enumerator.cs b/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Enumerator.cs index 614b2e52..957cb697 100644 --- a/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Enumerator.cs +++ b/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Enumerator.cs @@ -1,8 +1,8 @@ using System.Diagnostics.CodeAnalysis; -using System.IO.Compression; using System.Runtime.CompilerServices; using System.Text; using ExcelReader.Core.Reader.Internal; +using ExcelReader.Core.Reader.Zip; namespace ExcelReader.Core.Reader.Xlsb { @@ -24,7 +24,7 @@ public sealed class Enumerator : PooledStreamRowEnumerator, IExcelRowEnumerator private readonly bool[] _styleIsDate; private readonly int[] _sharedOffsets; private readonly Utf8StringCache? _contentCache; - private readonly ZipArchiveEntry? _entry; + private readonly ZipEntryRef _entry; private bool _ended; private bool _pendingRowHdr; @@ -35,26 +35,28 @@ internal Enumerator(XlsbReader reader, Stream sheet, long entryLength = 0, Cance _styleIsDate = reader._styleIsDate; _sharedOffsets = reader._sharedOffsets; _contentCache = reader._options.InternStrings ? new Utf8StringCache() : null; + _lease = reader.Lifetime; } - internal Enumerator(XlsbReader reader, ZipArchiveEntry entry, CancellationToken ct) - : base(reader._options.MaxCellBytes, nameof(ExcelReaderOptions.MaxCellBytes), WorkbookLookups.InitialBufferCapacity(entry.Length), ct) + internal Enumerator(XlsbReader reader, ZipEntryRef entry, CancellationToken ct) + : base(reader._options.MaxCellBytes, nameof(ExcelReaderOptions.MaxCellBytes), WorkbookLookups.InitialBufferCapacity(entry.UncompressedSize), ct) { _reader = reader; _styleIsDate = reader._styleIsDate; _sharedOffsets = reader._sharedOffsets; _contentCache = reader._options.InternStrings ? new Utf8StringCache() : null; _entry = entry; + _lease = reader.Lifetime; } private protected override Stream OpenSource() { - return WorkbookLookups.OpenEntryStream(_entry!, _reader._decompressedBytes, _reader._options); + return _reader._zip!.OpenEntryStream(_entry, _reader._decompressedBytes, _reader._options); } private protected override async ValueTask OpenSourceAsync() { - return await WorkbookLookups.OpenEntryStreamAsync(_entry!, _reader._decompressedBytes, _reader._options, _ct).ConfigureAwait(false); + return await _reader._zip!.OpenEntryStreamAsync(_entry, _reader._decompressedBytes, _reader._options, _ct).ConfigureAwait(false); } /// @@ -76,36 +78,25 @@ public ValueTask MoveNextAsync() { return new ValueTask(false); } - while (true) + ResetRow(); + if (!_pendingRowHdr) { - ResetRow(); - if (!_pendingRowHdr) - { - int seek = SeekRowHdrFromBuffer(); - if (seek == 0) - { - return new ValueTask(false); - } - if (seek == 2) - { - return MoveNextRowAsync(seekDone: false); - } - } - _pendingRowHdr = false; - int collect = CollectCellsFromBuffer(); - if (collect == 2) - { - return MoveNextRowAsync(seekDone: true); - } - if (_acc.Count > 0) + int seek = SeekRowHdrFromBuffer(); + if (seek == 0) { - return new ValueTask(true); + return new ValueTask(false); } - if (!_pendingRowHdr) + if (seek == 2) { - return new ValueTask(false); + return MoveNextRowAsync(seekDone: false); } } + _pendingRowHdr = false; + if (CollectCellsFromBuffer() == 2) + { + return MoveNextRowAsync(seekDone: true); + } + return new ValueTask(true); } private async ValueTask MoveNextRowAsync(bool seekDone) @@ -116,24 +107,7 @@ private async ValueTask MoveNextRowAsync(bool seekDone) } _pendingRowHdr = false; await CollectCellsAsync().ConfigureAwait(false); - while (true) - { - if (_acc.Count > 0) - { - return true; - } - if (!_pendingRowHdr) - { - return false; - } - ResetRow(); - if (!await SeekRowHdrAsync().ConfigureAwait(false)) - { - return false; - } - _pendingRowHdr = false; - await CollectCellsAsync().ConfigureAwait(false); - } + return true; } private bool MoveNextCore() @@ -142,24 +116,14 @@ private bool MoveNextCore() { return false; } - while (true) + ResetRow(); + if (!_pendingRowHdr && !SkipToRowHdr()) { - ResetRow(); - if (!_pendingRowHdr && !SkipToRowHdr()) - { - return false; - } - _pendingRowHdr = false; - CollectCells(); - if (_acc.Count > 0) - { - return true; - } - if (!_pendingRowHdr) - { - return false; - } + return false; } + _pendingRowHdr = false; + CollectCells(); + return true; } private bool SkipToRowHdr() diff --git a/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Memory.cs b/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Memory.cs deleted file mode 100644 index a2b18a18..00000000 --- a/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.Memory.cs +++ /dev/null @@ -1,55 +0,0 @@ -using ExcelReader.Core.Reader.Internal; -using ExcelReader.Core.Reader.Zip; - -namespace ExcelReader.Core.Reader.Xlsb -{ - public sealed partial class XlsbReader - { - internal static XlsbReader CreateFromMemory(ReadOnlyMemory data, ExcelReaderOptions? options = null) - { - ExcelReaderOptions effectiveOptions = options ?? ExcelReaderOptions.Default; - return CreateFromMemory(ZipMemoryIndex.Create(data, effectiveOptions), effectiveOptions); - } - - internal static XlsbReader CreateFromMemory(ZipMemoryIndex memZip, ExcelReaderOptions effectiveOptions) - { - return ZipReaderOpen.FromMemory(memZip, zip => BuildFromMemory(zip, effectiveOptions)); - } - - private static XlsbReader BuildFromMemory(ZipMemoryIndex memZip, ExcelReaderOptions effectiveOptions) - { - DecompressedByteCounter decompressedBytes = new(effectiveOptions.MaxTotalDecompressedBytes); - using ZipPart wbPart = memZip.OpenPartOrDefault("xl/workbook.bin"u8, decompressedBytes); - using ZipPart relsPart = memZip.OpenPartOrDefault("xl/_rels/workbook.bin.rels"u8, decompressedBytes); - (string Name, string Path, ExcelSheetVisibility Visibility)[] sheets = XlsbWorkbook.ParseSheets(wbPart.Memory.Span, relsPart.Memory.Span); - if (sheets.Length == 0) - { - throw new InvalidDataException("The workbook contains no sheets."); - } - using ZipPart stylesPart = memZip.OpenPartOrDefault("xl/styles.bin"u8, decompressedBytes); - bool[] styleIsDate = XlsbStyles.ParseStyleDateFlags(stylesPart.Memory.Span); - bool date1904 = XlsbWorkbook.ParseDate1904(wbPart.Memory.Span); - (byte[] flat, int[] offsets) = LoadSharedStringsFromMemory(memZip, decompressedBytes, effectiveOptions); - return new XlsbReader(memZip, sheets, styleIsDate, date1904, flat, offsets, effectiveOptions, decompressedBytes); - } - - private static (byte[] Flat, int[] Offsets) LoadSharedStringsFromMemory( - ZipMemoryIndex memZip, DecompressedByteCounter decompressedBytes, ExcelReaderOptions options) - { - if (!memZip.TryGetEntry("xl/sharedStrings.bin"u8, out ZipEntryRef entry)) - { - return ([], [0]); - } - WorkbookLookups.ThrowIfSharedEntryTooLarge(entry.UncompressedSize, decompressedBytes, options); - using ZipPart part = memZip.OpenPart(entry, decompressedBytes, - nameof(ExcelReaderOptions.MaxSharedStringBytes), options.MaxSharedStringBytes); - return XlsbSharedStrings.Parse(part.Memory.Span, options); - } - - private Enumerator GetEnumeratorFromMemory() - { - ZipEntryRef entry = WorkbookLookups.GetWorksheetEntry(_memZip!, _sheets![_current].Path); - return new Enumerator(this, _memZip!.OpenEntryStream(entry, _decompressedBytes, _options), entry.UncompressedSize); - } - } -} diff --git a/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.cs b/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.cs index 0eb69b3c..e46b52d2 100644 --- a/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.cs +++ b/src/ExcelReader.Core/Reader/Xlsb/XlsbReader.cs @@ -1,6 +1,6 @@ using System.Buffers; -using System.IO.Compression; using ExcelReader.Core.Reader.Internal; +using ExcelReader.Core.Reader.Sources; using ExcelReader.Core.Reader.Zip; namespace ExcelReader.Core.Reader.Xlsb @@ -21,16 +21,14 @@ public sealed partial class XlsbReader : IExcelRowReader, IExcelRowReader data, ExcelReaderOptions? options = null) { - _leaveOpen = true; - _memZip = memZip; - _options = options; - _decompressedBytes = decompressedBytes; - _sheets = sheets; - _styleIsDate = styleIsDate; - IsDate1904 = date1904; - _sharedFlat = sharedFlat; - _sharedOffsets = sharedOffsets; - _pooledSharedFlat = false; + ExcelReaderOptions effectiveOptions = options ?? ExcelReaderOptions.Default; + return new XlsbReader(ZipIndex.Create(data, effectiveOptions), effectiveOptions); } - internal static ValueTask CreateAsync(Stream stream, bool leaveOpen, ExcelReaderOptions? options = null, CancellationToken ct = default) + internal static XlsbReader CreateFromIndex(ZipIndex zip, ExcelReaderOptions options) { - ExcelReaderOptions effectiveOptions = options ?? ExcelReaderOptions.Default; - DecompressedByteCounter decompressedBytes = new(effectiveOptions.MaxTotalDecompressedBytes); - return ZipReaderOpen.OpenAsync(stream, leaveOpen, effectiveOptions, - zip => ParseAsync(stream, leaveOpen, zip, effectiveOptions, decompressedBytes, ct), ct); + return new XlsbReader(zip, options); } - internal static ValueTask CreateFromOpenZipAsync( - Stream stream, bool leaveOpen, ZipArchive zip, ExcelReaderOptions? options, CancellationToken ct) + internal static async ValueTask CreateAsync(Stream stream, bool leaveOpen, ExcelReaderOptions? options = null, CancellationToken ct = default) { ExcelReaderOptions effectiveOptions = options ?? ExcelReaderOptions.Default; - DecompressedByteCounter decompressedBytes = new(effectiveOptions.MaxTotalDecompressedBytes); - return ZipReaderOpen.FromOpenZipAsync(stream, leaveOpen, zip, effectiveOptions, - z => ParseAsync(stream, leaveOpen, z, effectiveOptions, decompressedBytes, ct)); + ByteSource source = await ByteSource.FromStreamAsync(stream, leaveOpen, ct).ConfigureAwait(false); + ZipIndex zip = await ZipIndex.CreateAsync(source, effectiveOptions, ct).ConfigureAwait(false); + return await CreateFromIndexAsync(zip, effectiveOptions, ct).ConfigureAwait(false); } - private static async ValueTask ParseAsync( - Stream stream, bool leaveOpen, ZipArchive zip, ExcelReaderOptions effectiveOptions, - DecompressedByteCounter decompressedBytes, CancellationToken ct) + internal static async ValueTask CreateFromIndexAsync(ZipIndex zip, ExcelReaderOptions options, CancellationToken ct) { - using ZipPart wbPart = await ZipEntryBytes.ReadAsync(zip, "xl/workbook.bin", decompressedBytes, ct).ConfigureAwait(false); - using ZipPart relsPart = await ZipEntryBytes.ReadAsync(zip, "xl/_rels/workbook.bin.rels", decompressedBytes, ct).ConfigureAwait(false); - var sheets = XlsbWorkbook.ParseSheets(wbPart.Memory.Span, relsPart.Memory.Span); - if (sheets.Length == 0) + try { - throw new InvalidDataException("The workbook contains no sheets."); + DecompressedByteCounter decompressedBytes = new(options.MaxTotalDecompressedBytes); + using ZipPart wbPart = await zip.OpenPartOrDefaultAsync("xl/workbook.bin"u8, decompressedBytes, ct).ConfigureAwait(false); + using ZipPart relsPart = await zip.OpenPartOrDefaultAsync("xl/_rels/workbook.bin.rels"u8, decompressedBytes, ct).ConfigureAwait(false); + (string Name, string Path, ExcelSheetVisibility Visibility)[] sheets = XlsbWorkbook.ParseSheets(wbPart.Memory.Span, relsPart.Memory.Span); + if (sheets.Length == 0) + { + throw new InvalidDataException("The workbook contains no sheets."); + } + using ZipPart stylesPart = await zip.OpenPartOrDefaultAsync("xl/styles.bin"u8, decompressedBytes, ct).ConfigureAwait(false); + bool[] styleIsDate = XlsbStyles.ParseStyleDateFlags(stylesPart.Memory.Span); + bool date1904 = XlsbWorkbook.ParseDate1904(wbPart.Memory.Span); + (byte[] flat, int[] offsets, bool pooled) = await LoadSharedStringsAsync(zip, decompressedBytes, options, ct).ConfigureAwait(false); + return new XlsbReader(zip, sheets, styleIsDate, date1904, flat, offsets, pooled, options, decompressedBytes); + } + catch + { + zip.Dispose(); + throw; } - using ZipPart stylesPart = await ZipEntryBytes.ReadAsync(zip, "xl/styles.bin", decompressedBytes, ct).ConfigureAwait(false); - var styleIsDate = XlsbStyles.ParseStyleDateFlags(stylesPart.Memory.Span); - bool date1904 = XlsbWorkbook.ParseDate1904(wbPart.Memory.Span); - var (flat, offsets) = await LoadSharedStringsAsync(zip, decompressedBytes, effectiveOptions, ct).ConfigureAwait(false); - return new XlsbReader(stream, leaveOpen, zip, sheets, styleIsDate, date1904, flat, offsets, effectiveOptions, decompressedBytes); } - private (byte[] Flat, int[] Offsets) LoadSharedStrings(ZipArchive zip) + private static (byte[] Flat, int[] Offsets, bool Pooled) LoadSharedStrings( + ZipIndex zip, DecompressedByteCounter decompressedBytes, ExcelReaderOptions options) { - ZipArchiveEntry? entry = zip.GetEntry("xl/sharedStrings.bin"); - if (entry is null) + if (!zip.TryGetEntry("xl/sharedStrings.bin"u8, out ZipEntryRef entry)) { - return ([], [0]); + return ([], [0], false); } - WorkbookLookups.ThrowIfSharedEntryTooLarge(entry.Length, _decompressedBytes, _options); - using LimitedReadStream stream = WorkbookLookups.OpenEntryStream(entry, _decompressedBytes, _options, - nameof(ExcelReaderOptions.MaxSharedStringBytes), _options.MaxSharedStringBytes); - return XlsbSharedStrings.ParseStreaming(stream, entry.Length, _options); + WorkbookLookups.ThrowIfSharedEntryTooLarge(entry.UncompressedSize, decompressedBytes, options); + if (zip.HasMemory) + { + using ZipPart part = zip.OpenPart(entry, decompressedBytes, + nameof(ExcelReaderOptions.MaxSharedStringBytes), options.MaxSharedStringBytes); + (byte[] parsedFlat, int[] parsedOffsets) = XlsbSharedStrings.Parse(part.Memory.Span, options); + return (parsedFlat, parsedOffsets, false); + } + using LimitedReadStream stream = zip.OpenEntryStream(entry, decompressedBytes, options, + nameof(ExcelReaderOptions.MaxSharedStringBytes), options.MaxSharedStringBytes); + (byte[] flat, int[] offsets) = XlsbSharedStrings.ParseStreaming(stream, entry.UncompressedSize, options); + return (flat, offsets, flat.Length != 0); } - private static async ValueTask<(byte[] Flat, int[] Offsets)> LoadSharedStringsAsync( - ZipArchive zip, DecompressedByteCounter decompressedBytes, ExcelReaderOptions options, CancellationToken ct) + private static async ValueTask<(byte[] Flat, int[] Offsets, bool Pooled)> LoadSharedStringsAsync( + ZipIndex zip, DecompressedByteCounter decompressedBytes, ExcelReaderOptions options, CancellationToken ct) { - ZipArchiveEntry? entry = zip.GetEntry("xl/sharedStrings.bin"); - if (entry is null) + if (zip.HasMemory || !zip.TryGetEntry("xl/sharedStrings.bin"u8, out ZipEntryRef entry)) { - return ([], [0]); + return LoadSharedStrings(zip, decompressedBytes, options); } - WorkbookLookups.ThrowIfSharedEntryTooLarge(entry.Length, decompressedBytes, options); - LimitedReadStream stream = await WorkbookLookups.OpenEntryStreamAsync(entry, decompressedBytes, options, ct, + WorkbookLookups.ThrowIfSharedEntryTooLarge(entry.UncompressedSize, decompressedBytes, options); + LimitedReadStream stream = await zip.OpenEntryStreamAsync(entry, decompressedBytes, options, ct, nameof(ExcelReaderOptions.MaxSharedStringBytes), options.MaxSharedStringBytes).ConfigureAwait(false); await using (stream.ConfigureAwait(false)) { - return await XlsbSharedStrings.ParseStreamingAsync(stream, entry.Length, options, ct).ConfigureAwait(false); + (byte[] flat, int[] offsets) = await XlsbSharedStrings.ParseStreamingAsync(stream, entry.UncompressedSize, options, ct).ConfigureAwait(false); + return (flat, offsets, flat.Length != 0); } } @@ -219,7 +210,13 @@ public void MoveToSheet(int index) internal ReadOnlySpan SharedSpan => _sharedFlat; - internal string?[] SharedStringCache => _sharedStringCache ??= WorkbookLookups.CreateSharedStringCache(_sharedOffsets); + internal string?[] SharedStringCache => Volatile.Read(ref _sharedStringCache) ?? CreateSharedStringCache(); + + private string?[] CreateSharedStringCache() + { + string?[] created = WorkbookLookups.CreateSharedStringCache(_sharedOffsets); + return Interlocked.CompareExchange(ref _sharedStringCache, created, null) ?? created; + } internal bool IsDateStyle(int style) { @@ -229,12 +226,17 @@ internal bool IsDateStyle(int style) /// public Enumerator GetEnumerator() { - if (_memZip is not null) + Lifetime.Acquire(this); + try + { + ZipEntryRef entry = WorkbookLookups.GetWorksheetEntry(_zip!, _sheets![_current].Path); + return new Enumerator(this, _zip!.OpenEntryStream(entry, _decompressedBytes, _options), entry.UncompressedSize); + } + catch { - return GetEnumeratorFromMemory(); + Lifetime.Release(); + throw; } - var entry = WorkbookLookups.GetWorksheetEntry(_zip!, _sheets![_current].Path); - return new Enumerator(this, WorkbookLookups.OpenEntryStream(entry, _decompressedBytes, _options), entry.Length); } IExcelRowEnumerator IExcelRowReader.GetEnumerator() @@ -245,11 +247,20 @@ IExcelRowEnumerator IExcelRowReader.GetEnumerator() /// public Enumerator GetAsyncEnumerator(CancellationToken ct = default) { - if (_memZip is not null) + if (_zip!.HasMemory) + { + return GetEnumerator(); + } + Lifetime.Acquire(this); + try { - return GetEnumeratorFromMemory(); + return new Enumerator(this, WorkbookLookups.GetWorksheetEntry(_zip, _sheets![_current].Path), ct); + } + catch + { + Lifetime.Release(); + throw; } - return new Enumerator(this, WorkbookLookups.GetWorksheetEntry(_zip!, _sheets![_current].Path), ct); } IExcelRowEnumerator IExcelRowReader.GetAsyncEnumerator(CancellationToken ct) @@ -261,42 +272,23 @@ IExcelRowEnumerator IExcelRowReader.GetAsyncEnumerator(Canc /// public void Dispose() { - if (Interlocked.Exchange(ref _disposed, 1) != 0) - { - return; - } - if (_pooledSharedFlat) - { - ArrayPool.Shared.Return(_sharedFlat); - } - _memZip?.Dispose(); - _zip?.Dispose(); - if (!_leaveOpen) - { - _stream?.Dispose(); - } + Lifetime.Close(); } /// - public async ValueTask DisposeAsync() + public ValueTask DisposeAsync() + { + Lifetime.Close(); + return ValueTask.CompletedTask; + } + + private void ReleaseResources() { - if (Interlocked.Exchange(ref _disposed, 1) != 0) - { - return; - } if (_pooledSharedFlat) { ArrayPool.Shared.Return(_sharedFlat); } - _memZip?.Dispose(); - if (_zip is not null) - { - await _zip.DisposeAsync().ConfigureAwait(false); - } - if (!_leaveOpen && _stream is not null) - { - await _stream.DisposeAsync().ConfigureAwait(false); - } + _zip?.Dispose(); } } diff --git a/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Enumerator.cs b/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Enumerator.cs index 05a39cc5..9b54bf4d 100644 --- a/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Enumerator.cs +++ b/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Enumerator.cs @@ -1,9 +1,9 @@ using System.Buffers.Text; using System.Diagnostics.CodeAnalysis; using System.Globalization; -using System.IO.Compression; using System.Runtime.CompilerServices; using ExcelReader.Core.Reader.Internal; +using ExcelReader.Core.Reader.Zip; using ExcelReader.Core.Writer.Internal; namespace ExcelReader.Core.Reader.Xlsx @@ -21,7 +21,7 @@ public sealed class Enumerator : PooledStreamRowEnumerator, IExcelRowEnumerator private readonly bool[] _styleIsDate; private int[] _sharedOffsets; private readonly Utf8StringCache? _contentCache; - private readonly ZipArchiveEntry? _entry; + private readonly ZipEntryRef _entry; private int _nextCol; private NsTokens? _ns; @@ -38,30 +38,32 @@ internal Enumerator(XlsxReader reader, Stream sheet, long entryLength = 0, Cance _styleIsDate = reader._styleIsDate; _sharedOffsets = reader._sharedOffsets; _contentCache = reader._options.InternStrings ? new Utf8StringCache() : null; + _lease = reader.Lifetime; } - internal Enumerator(XlsxReader reader, ZipArchiveEntry entry, CancellationToken ct) - : base(reader._options.MaxCellBytes, nameof(ExcelReaderOptions.MaxCellBytes), WorkbookLookups.InitialBufferCapacity(entry.Length), ct) + internal Enumerator(XlsxReader reader, ZipEntryRef entry, CancellationToken ct) + : base(reader._options.MaxCellBytes, nameof(ExcelReaderOptions.MaxCellBytes), WorkbookLookups.InitialBufferCapacity(entry.UncompressedSize), ct) { _reader = reader; _styleIsDate = reader._styleIsDate; _sharedOffsets = reader._sharedOffsets; _contentCache = reader._options.InternStrings ? new Utf8StringCache() : null; _entry = entry; + _lease = reader.Lifetime; } private protected override Stream OpenSource() { _reader.EnsureSharedLoaded(); _sharedOffsets = _reader._sharedOffsets; - return WorkbookLookups.OpenEntryStream(_entry!, _reader._decompressedBytes, _reader._options); + return _reader._zip.OpenEntryStream(_entry, _reader._decompressedBytes, _reader._options); } private protected override async ValueTask OpenSourceAsync() { await _reader.EnsureSharedLoadedAsync(_ct).ConfigureAwait(false); _sharedOffsets = _reader._sharedOffsets; - return await WorkbookLookups.OpenEntryStreamAsync(_entry!, _reader._decompressedBytes, _reader._options, _ct).ConfigureAwait(false); + return await _reader._zip.OpenEntryStreamAsync(_entry, _reader._decompressedBytes, _reader._options, _ct).ConfigureAwait(false); } /// diff --git a/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Loading.cs b/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Loading.cs index 2d24b6a9..c7de2bb8 100644 --- a/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Loading.cs +++ b/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Loading.cs @@ -1,5 +1,4 @@ using System.Buffers; -using System.IO.Compression; using System.Text; using ExcelReader.Core.Reader.Internal; using ExcelReader.Core.Reader.Zip; @@ -71,41 +70,56 @@ private static bool ParseDate1904(ReadOnlySpan src) private void EnsureSharedLoaded() { - if (_sharedLoaded) + if (_sharedGate.IsDone) { return; } - _sharedLoaded = true; - ZipArchiveEntry? entry = _zip!.GetEntry("xl/sharedStrings.xml"); - if (entry is null) + _sharedGate.Run(this, static reader => reader.LoadShared(), static reader => reader.ReturnSharedTable()); + } + + private ValueTask EnsureSharedLoadedAsync(CancellationToken ct) + { + if (_sharedGate.IsDone) { - return; + return ValueTask.CompletedTask; } - WorkbookLookups.ThrowIfSharedEntryTooLarge(entry.Length, _decompressedBytes, _options); - using LimitedReadStream stream = WorkbookLookups.OpenEntryStream(entry, _decompressedBytes, _options, - nameof(ExcelReaderOptions.MaxSharedStringBytes), _options.MaxSharedStringBytes); - ParseSharedStreaming(stream, entry.Length); + return _sharedGate.RunAsync( + this, static (reader, token) => reader.LoadSharedAsync(token), static reader => reader.ReturnSharedTable(), ct); } - private async ValueTask EnsureSharedLoadedAsync(CancellationToken ct) + private void LoadShared() { - if (_sharedLoaded) + if (!_zip.TryGetEntry("xl/sharedStrings.xml"u8, out ZipEntryRef entry)) + { + return; + } + WorkbookLookups.ThrowIfSharedEntryTooLarge(entry.UncompressedSize, _decompressedBytes, _options); + if (_zip.HasMemory) { + using ZipPart part = _zip.OpenPart(entry, _decompressedBytes, + nameof(ExcelReaderOptions.MaxSharedStringBytes), _options.MaxSharedStringBytes); + ParseSharedFromMemory(part.Memory, entry.UncompressedSize); return; } - _sharedLoaded = true; - ZipArchiveEntry? entry = _zip!.GetEntry("xl/sharedStrings.xml"); - if (entry is null) + using LimitedReadStream stream = _zip.OpenEntryStream(entry, _decompressedBytes, _options, + nameof(ExcelReaderOptions.MaxSharedStringBytes), _options.MaxSharedStringBytes); + ParseSharedStreaming(stream, entry.UncompressedSize); + } + + private async ValueTask LoadSharedAsync(CancellationToken ct) + { + if (_zip.HasMemory || !_zip.TryGetEntry("xl/sharedStrings.xml"u8, out ZipEntryRef entry)) { + LoadShared(); return; } - WorkbookLookups.ThrowIfSharedEntryTooLarge(entry.Length, _decompressedBytes, _options); - LimitedReadStream stream = await WorkbookLookups.OpenEntryStreamAsync( + WorkbookLookups.ThrowIfSharedEntryTooLarge(entry.UncompressedSize, _decompressedBytes, _options); + LimitedReadStream stream = await _zip.OpenEntryStreamAsync( entry, _decompressedBytes, _options, ct, nameof(ExcelReaderOptions.MaxSharedStringBytes), _options.MaxSharedStringBytes).ConfigureAwait(false); await using (stream.ConfigureAwait(false)) { - await ParseSharedStreamingAsync(stream, entry.Length, ct).ConfigureAwait(false); + await ParseSharedStreamingAsync(stream, entry.UncompressedSize, ct).ConfigureAwait(false); } } diff --git a/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Memory.cs b/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Memory.cs index 87f1f50e..64672c4a 100644 --- a/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Memory.cs +++ b/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.Memory.cs @@ -6,50 +6,6 @@ namespace ExcelReader.Core.Reader.Xlsx { public sealed partial class XlsxReader { - internal static XlsxReader CreateFromMemory(ReadOnlyMemory data, ExcelReaderOptions? options = null) - { - ExcelReaderOptions effectiveOptions = options ?? ExcelReaderOptions.Default; - return CreateFromMemory(ZipMemoryIndex.Create(data, effectiveOptions), effectiveOptions); - } - - internal static XlsxReader CreateFromMemory(ZipMemoryIndex memZip, ExcelReaderOptions effectiveOptions) - { - return ZipReaderOpen.FromMemory(memZip, zip => BuildFromMemory(zip, effectiveOptions)); - } - - private static XlsxReader BuildFromMemory(ZipMemoryIndex memZip, ExcelReaderOptions effectiveOptions) - { - DecompressedByteCounter decompressedBytes = new(effectiveOptions.MaxTotalDecompressedBytes); - using ZipPart wbPart = memZip.OpenPartOrDefault("xl/workbook.xml"u8, decompressedBytes); - using ZipPart relsPart = memZip.OpenPartOrDefault("xl/_rels/workbook.xml.rels"u8, decompressedBytes); - (string Name, string Path, ExcelSheetVisibility Visibility)[] sheets = ParseSheets(wbPart.Memory.Span, relsPart.Memory.Span); - if (sheets.Length == 0) - { - throw new InvalidDataException("The workbook contains no sheets."); - } - using ZipPart stylesPart = memZip.OpenPartOrDefault("xl/styles.xml"u8, decompressedBytes); - bool[] styleIsDate = ParseStyleDateFlags(stylesPart.Memory.Span); - bool date1904 = ParseDate1904(wbPart.Memory.Span); - return new XlsxReader(memZip, sheets, styleIsDate, date1904, effectiveOptions, decompressedBytes); - } - - private void EnsureSharedLoadedFromMemory() - { - if (_sharedLoaded) - { - return; - } - _sharedLoaded = true; - if (!_memZip!.TryGetEntry("xl/sharedStrings.xml"u8, out ZipEntryRef entry)) - { - return; - } - WorkbookLookups.ThrowIfSharedEntryTooLarge(entry.UncompressedSize, _decompressedBytes, _options); - using ZipPart part = _memZip.OpenPart(entry, _decompressedBytes, - nameof(ExcelReaderOptions.MaxSharedStringBytes), _options.MaxSharedStringBytes); - ParseSharedFromMemory(part.Memory, entry.UncompressedSize); - } - private void ParseSharedFromMemory(ReadOnlyMemory content, long entryLength) { LimitChecks.ThrowIfEntryLengthExceeds(entryLength, Array.MaxLength, "ArrayMaxLength"); @@ -59,11 +15,5 @@ private void ParseSharedFromMemory(ReadOnlyMemory content, long entryLengt _sharedFlat = ArrayPool.Shared.Rent(Math.Max(1, partLength)); _sharedOffsets = ParseSharedBody(io, stream: null, partLength); } - - private Enumerator GetEnumeratorFromMemory() - { - ZipEntryRef entry = WorkbookLookups.GetWorksheetEntry(_memZip!, _sheets[_current].Path); - return new Enumerator(this, _memZip!.OpenEntryStream(entry, _decompressedBytes, _options), entry.UncompressedSize); - } } } diff --git a/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.cs b/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.cs index 5c9c9792..65f65499 100644 --- a/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.cs +++ b/src/ExcelReader.Core/Reader/Xlsx/XlsxReader.cs @@ -1,6 +1,7 @@ using System.Buffers; -using System.IO.Compression; +using System.Diagnostics.CodeAnalysis; using ExcelReader.Core.Reader.Internal; +using ExcelReader.Core.Reader.Sources; using ExcelReader.Core.Reader.Zip; namespace ExcelReader.Core.Reader.Xlsx @@ -8,64 +9,57 @@ namespace ExcelReader.Core.Reader.Xlsx /// Reads rows from an Office Open XML (.xlsx) workbook, streaming each sheet's cells without loading the whole file into memory. public sealed partial class XlsxReader : IExcelRowReader, IExcelRowReader { - private readonly Stream? _stream; - private readonly bool _leaveOpen; - private readonly ZipArchive? _zip; - private readonly ZipMemoryIndex? _memZip; + private readonly ZipIndex _zip; private readonly ExcelReaderOptions _options; private readonly DecompressedByteCounter _decompressedBytes; private readonly (string Name, string Path, ExcelSheetVisibility Visibility)[] _sheets; private readonly bool[] _styleIsDate; private int _current; + [SuppressMessage("Usage", "CA2213:Disposable fields should be disposed", Justification = "Disposed by ReleaseResources, which the lifetime runs after the last enumerator.")] + private readonly OnceGate _sharedGate = new(); private byte[] _sharedFlat = []; private int[] _sharedOffsets = [0]; - private bool _sharedLoaded; private string?[]? _sharedStringCache; + internal ReaderLifetime Lifetime { get; } + internal XlsxReader(Stream stream, bool leaveOpen, ExcelReaderOptions? options = null) - : this(stream, leaveOpen, ZipReaderOpen.Open(stream, leaveOpen), options) + : this(ZipIndex.Create(ByteSource.FromStream(stream, leaveOpen), options ?? ExcelReaderOptions.Default), options ?? ExcelReaderOptions.Default) { } - internal XlsxReader(Stream stream, bool leaveOpen, ZipArchive zip, ExcelReaderOptions? options = null) + private XlsxReader(ZipIndex zip, ExcelReaderOptions options) { - _stream = stream; - _leaveOpen = leaveOpen; - _options = options ?? ExcelReaderOptions.Default; - _decompressedBytes = new DecompressedByteCounter(_options.MaxTotalDecompressedBytes); + Lifetime = new ReaderLifetime(ReleaseResources); _zip = zip; + _options = options; + _decompressedBytes = new DecompressedByteCounter(options.MaxTotalDecompressedBytes); try { - LimitChecks.ThrowIfTooManyEntries(_zip.Entries.Count, _options); - using ZipPart wbPart = ZipEntryBytes.Read(_zip, "xl/workbook.xml", _decompressedBytes); - using ZipPart relsPart = ZipEntryBytes.Read(_zip, "xl/_rels/workbook.xml.rels", _decompressedBytes); + using ZipPart wbPart = zip.OpenPartOrDefault("xl/workbook.xml"u8, _decompressedBytes); + using ZipPart relsPart = zip.OpenPartOrDefault("xl/_rels/workbook.xml.rels"u8, _decompressedBytes); _sheets = ParseSheets(wbPart.Memory.Span, relsPart.Memory.Span); if (_sheets.Length == 0) { throw new InvalidDataException("The workbook contains no sheets."); } - using ZipPart stylesPart = ZipEntryBytes.Read(_zip, "xl/styles.xml", _decompressedBytes); + using ZipPart stylesPart = zip.OpenPartOrDefault("xl/styles.xml"u8, _decompressedBytes); _styleIsDate = ParseStyleDateFlags(stylesPart.Memory.Span); IsDate1904 = ParseDate1904(wbPart.Memory.Span); } catch { - _zip.Dispose(); - if (!leaveOpen) - { - stream.Dispose(); - } + zip.Dispose(); throw; } } - private XlsxReader(Stream stream, bool leaveOpen, ZipArchive zip, + private XlsxReader(ZipIndex zip, (string Name, string Path, ExcelSheetVisibility Visibility)[] sheets, bool[] styleIsDate, bool date1904, ExcelReaderOptions options, DecompressedByteCounter decompressedBytes) { - _stream = stream; - _leaveOpen = leaveOpen; + Lifetime = new ReaderLifetime(ReleaseResources); _zip = zip; _options = options; _decompressedBytes = decompressedBytes; @@ -74,51 +68,47 @@ private XlsxReader(Stream stream, bool leaveOpen, ZipArchive zip, IsDate1904 = date1904; } - private XlsxReader(ZipMemoryIndex memZip, - (string Name, string Path, ExcelSheetVisibility Visibility)[] sheets, bool[] styleIsDate, bool date1904, - ExcelReaderOptions options, DecompressedByteCounter decompressedBytes) + internal static XlsxReader CreateFromMemory(ReadOnlyMemory data, ExcelReaderOptions? options = null) { - _leaveOpen = true; - _memZip = memZip; - _options = options; - _decompressedBytes = decompressedBytes; - _sheets = sheets; - _styleIsDate = styleIsDate; - IsDate1904 = date1904; + ExcelReaderOptions effectiveOptions = options ?? ExcelReaderOptions.Default; + return new XlsxReader(ZipIndex.Create(data, effectiveOptions), effectiveOptions); } - internal static ValueTask CreateAsync(Stream stream, bool leaveOpen, ExcelReaderOptions? options = null, CancellationToken ct = default) + internal static XlsxReader CreateFromIndex(ZipIndex zip, ExcelReaderOptions options) { - ExcelReaderOptions effectiveOptions = options ?? ExcelReaderOptions.Default; - DecompressedByteCounter decompressedBytes = new(effectiveOptions.MaxTotalDecompressedBytes); - return ZipReaderOpen.OpenAsync(stream, leaveOpen, effectiveOptions, - zip => ParseAsync(stream, leaveOpen, zip, effectiveOptions, decompressedBytes, ct), ct); + return new XlsxReader(zip, options); } - internal static ValueTask CreateFromOpenZipAsync( - Stream stream, bool leaveOpen, ZipArchive zip, ExcelReaderOptions? options, CancellationToken ct) + internal static async ValueTask CreateAsync(Stream stream, bool leaveOpen, ExcelReaderOptions? options = null, CancellationToken ct = default) { ExcelReaderOptions effectiveOptions = options ?? ExcelReaderOptions.Default; - DecompressedByteCounter decompressedBytes = new(effectiveOptions.MaxTotalDecompressedBytes); - return ZipReaderOpen.FromOpenZipAsync(stream, leaveOpen, zip, effectiveOptions, - z => ParseAsync(stream, leaveOpen, z, effectiveOptions, decompressedBytes, ct)); + ByteSource source = await ByteSource.FromStreamAsync(stream, leaveOpen, ct).ConfigureAwait(false); + ZipIndex zip = await ZipIndex.CreateAsync(source, effectiveOptions, ct).ConfigureAwait(false); + return await CreateFromIndexAsync(zip, effectiveOptions, ct).ConfigureAwait(false); } - private static async ValueTask ParseAsync( - Stream stream, bool leaveOpen, ZipArchive zip, ExcelReaderOptions effectiveOptions, - DecompressedByteCounter decompressedBytes, CancellationToken ct) + internal static async ValueTask CreateFromIndexAsync(ZipIndex zip, ExcelReaderOptions options, CancellationToken ct) { - using ZipPart wbPart = await ZipEntryBytes.ReadAsync(zip, "xl/workbook.xml", decompressedBytes, ct).ConfigureAwait(false); - using ZipPart relsPart = await ZipEntryBytes.ReadAsync(zip, "xl/_rels/workbook.xml.rels", decompressedBytes, ct).ConfigureAwait(false); - var sheets = ParseSheets(wbPart.Memory.Span, relsPart.Memory.Span); - if (sheets.Length == 0) + try { - throw new InvalidDataException("The workbook contains no sheets."); + DecompressedByteCounter decompressedBytes = new(options.MaxTotalDecompressedBytes); + using ZipPart wbPart = await zip.OpenPartOrDefaultAsync("xl/workbook.xml"u8, decompressedBytes, ct).ConfigureAwait(false); + using ZipPart relsPart = await zip.OpenPartOrDefaultAsync("xl/_rels/workbook.xml.rels"u8, decompressedBytes, ct).ConfigureAwait(false); + (string Name, string Path, ExcelSheetVisibility Visibility)[] sheets = ParseSheets(wbPart.Memory.Span, relsPart.Memory.Span); + if (sheets.Length == 0) + { + throw new InvalidDataException("The workbook contains no sheets."); + } + using ZipPart stylesPart = await zip.OpenPartOrDefaultAsync("xl/styles.xml"u8, decompressedBytes, ct).ConfigureAwait(false); + bool[] styleIsDate = ParseStyleDateFlags(stylesPart.Memory.Span); + bool date1904 = ParseDate1904(wbPart.Memory.Span); + return new XlsxReader(zip, sheets, styleIsDate, date1904, options, decompressedBytes); + } + catch + { + zip.Dispose(); + throw; } - using ZipPart stylesPart = await ZipEntryBytes.ReadAsync(zip, "xl/styles.xml", decompressedBytes, ct).ConfigureAwait(false); - var styleIsDate = ParseStyleDateFlags(stylesPart.Memory.Span); - bool date1904 = ParseDate1904(wbPart.Memory.Span); - return new XlsxReader(stream, leaveOpen, zip, sheets, styleIsDate, date1904, effectiveOptions, decompressedBytes); } /// @@ -168,14 +158,18 @@ public void MoveToSheet(int index) /// public Enumerator GetEnumerator() { - if (_memZip is not null) + Lifetime.Acquire(this); + try + { + EnsureSharedLoaded(); + ZipEntryRef entry = WorkbookLookups.GetWorksheetEntry(_zip, _sheets[_current].Path); + return new Enumerator(this, _zip.OpenEntryStream(entry, _decompressedBytes, _options), entry.UncompressedSize); + } + catch { - EnsureSharedLoadedFromMemory(); - return GetEnumeratorFromMemory(); + Lifetime.Release(); + throw; } - EnsureSharedLoaded(); - var entry = WorkbookLookups.GetWorksheetEntry(_zip!, _sheets[_current].Path); - return new Enumerator(this, WorkbookLookups.OpenEntryStream(entry, _decompressedBytes, _options), entry.Length); } IExcelRowEnumerator IExcelRowReader.GetEnumerator() @@ -186,12 +180,20 @@ IExcelRowEnumerator IExcelRowReader.GetEnumerator() /// public Enumerator GetAsyncEnumerator(CancellationToken ct = default) { - if (_memZip is not null) + if (_zip.HasMemory) { - EnsureSharedLoadedFromMemory(); - return GetEnumeratorFromMemory(); + return GetEnumerator(); + } + Lifetime.Acquire(this); + try + { + return new Enumerator(this, WorkbookLookups.GetWorksheetEntry(_zip, _sheets[_current].Path), ct); + } + catch + { + Lifetime.Release(); + throw; } - return new Enumerator(this, WorkbookLookups.GetWorksheetEntry(_zip!, _sheets[_current].Path), ct); } IExcelRowEnumerator IExcelRowReader.GetAsyncEnumerator(CancellationToken ct) @@ -201,41 +203,42 @@ IExcelRowEnumerator IExcelRowReader.GetAsyncEnumerator(Canc internal ReadOnlySpan SharedSpan => _sharedFlat; - internal string?[] SharedStringCache => _sharedStringCache ??= WorkbookLookups.CreateSharedStringCache(_sharedOffsets); + internal string?[] SharedStringCache => Volatile.Read(ref _sharedStringCache) ?? CreateSharedStringCache(); + + private string?[] CreateSharedStringCache() + { + string?[] created = WorkbookLookups.CreateSharedStringCache(_sharedOffsets); + return Interlocked.CompareExchange(ref _sharedStringCache, created, null) ?? created; + } /// public void Dispose() { - if (_sharedFlat.Length > 0) - { - ArrayPool.Shared.Return(_sharedFlat); - _sharedFlat = []; - } - _memZip?.Dispose(); - _zip?.Dispose(); - if (!_leaveOpen) - { - _stream?.Dispose(); - } + Lifetime.Close(); } /// - public async ValueTask DisposeAsync() + public ValueTask DisposeAsync() + { + Lifetime.Close(); + return ValueTask.CompletedTask; + } + + private void ReleaseResources() + { + ReturnSharedTable(); + _sharedGate.Dispose(); + _zip.Dispose(); + } + + private void ReturnSharedTable() { if (_sharedFlat.Length > 0) { ArrayPool.Shared.Return(_sharedFlat); - _sharedFlat = []; - } - _memZip?.Dispose(); - if (_zip is not null) - { - await _zip.DisposeAsync().ConfigureAwait(false); - } - if (!_leaveOpen && _stream is not null) - { - await _stream.DisposeAsync().ConfigureAwait(false); } + _sharedFlat = []; + _sharedOffsets = [0]; } } } diff --git a/src/ExcelReader.Core/Reader/Zip/LimitedReadStream.cs b/src/ExcelReader.Core/Reader/Zip/LimitedReadStream.cs index 8bfb9641..185a25ba 100644 --- a/src/ExcelReader.Core/Reader/Zip/LimitedReadStream.cs +++ b/src/ExcelReader.Core/Reader/Zip/LimitedReadStream.cs @@ -18,15 +18,14 @@ internal void Add(long bytes) { return; } - long total = checked(_total + bytes); - if (total > _limit) + long total = Interlocked.Add(ref _total, bytes); + if (total > _limit || total < 0) { - throw new ExcelLimitExceededException(_limitName, _limit, total); + throw new ExcelLimitExceededException(_limitName, _limit, total < 0 ? long.MaxValue : total); } - _total = total; } - internal long Remaining => _limit <= 0 ? long.MaxValue : _limit - _total; + internal long Remaining => _limit <= 0 ? long.MaxValue : _limit - Interlocked.Read(ref _total); } internal sealed class LimitedReadStream : Stream diff --git a/src/ExcelReader.Core/Reader/Zip/ZipEntryBytes.cs b/src/ExcelReader.Core/Reader/Zip/ZipEntryBytes.cs deleted file mode 100644 index 270c9557..00000000 --- a/src/ExcelReader.Core/Reader/Zip/ZipEntryBytes.cs +++ /dev/null @@ -1,121 +0,0 @@ -using System.Buffers; -using System.IO.Compression; -using ExcelReader.Core.Reader.Internal; - -namespace ExcelReader.Core.Reader.Zip -{ - internal static class ZipEntryBytes - { - private static void ThrowIfEntryLengthExceedsLimits( - ZipArchiveEntry entry, DecompressedByteCounter counter, string entryLimitName, long entryLimit) - { - LimitChecks.ThrowIfEntryLengthExceeds(entry.Length, counter.Remaining, - nameof(ExcelReaderOptions.MaxTotalDecompressedBytes)); - if (entryLimit > 0) - { - LimitChecks.ThrowIfEntryLengthExceeds(entry.Length, entryLimit, entryLimitName); - } - } - - internal static ZipPart Read( - ZipArchive zip, - string name, - DecompressedByteCounter counter, - string entryLimitName = "", - long entryLimit = 0) - { - ZipArchiveEntry? entry = zip.GetEntry(name); - return entry is null ? default : Read(entry, counter, entryLimitName, entryLimit); - } - - internal static ZipPart Read( - ZipArchiveEntry entry, - DecompressedByteCounter counter, - string entryLimitName = "", - long entryLimit = 0) - { - ThrowIfEntryLengthExceedsLimits(entry, counter, entryLimitName, entryLimit); - using var stream = new LimitedReadStream(entry.Open(), counter, entryLimitName, entryLimit); - int length = checked((int)entry.Length); - byte[] rented = ArrayPool.Shared.Rent(length); - try - { - ReadExactlyChecked(stream, rented.AsSpan(0, length)); - } - catch - { - ArrayPool.Shared.Return(rented); - throw; - } - return new ZipPart(rented.AsMemory(0, length), rented); - } - - private static void ReadExactlyChecked(Stream stream, Span destination) - { - try - { - stream.ReadExactly(destination); - } - catch (EndOfStreamException ex) - { - throw new InvalidDataException("The ZIP entry produced less data than its declared uncompressed size.", ex); - } - } - - internal static async ValueTask ReadAsync( - ZipArchive zip, - string name, - DecompressedByteCounter counter, - CancellationToken ct, - string entryLimitName = "", - long entryLimit = 0) - { - ZipArchiveEntry? entry = zip.GetEntry(name); - if (entry is null) - { - return default; - } - return await ReadAsync(entry, counter, ct, entryLimitName, entryLimit).ConfigureAwait(false); - } - - internal static async ValueTask ReadAsync( - ZipArchiveEntry entry, - DecompressedByteCounter counter, - CancellationToken ct, - string entryLimitName = "", - long entryLimit = 0) - { - ThrowIfEntryLengthExceedsLimits(entry, counter, entryLimitName, entryLimit); - Stream opened = await entry.OpenAsync(ct).ConfigureAwait(false); - var stream = new LimitedReadStream(opened, counter, entryLimitName, entryLimit); - await using (stream.ConfigureAwait(false)) - { - int length = checked((int)entry.Length); - byte[] rented = ArrayPool.Shared.Rent(length); - try - { - await ReadExactlyCheckedAsync(stream, rented.AsMemory(0, length), ct).ConfigureAwait(false); - } - catch - { - ArrayPool.Shared.Return(rented); - throw; - } - return new ZipPart(rented.AsMemory(0, length), rented); - } - } - - private static async ValueTask ReadExactlyCheckedAsync(Stream stream, Memory destination, CancellationToken ct) - { - try - { - await stream.ReadExactlyAsync(destination, ct).ConfigureAwait(false); - } - catch (EndOfStreamException ex) - { - throw new InvalidDataException("The ZIP entry produced less data than its declared uncompressed size.", ex); - } - } - - } -} diff --git a/src/ExcelReader.Core/Reader/Zip/ZipMemoryIndex.cs b/src/ExcelReader.Core/Reader/Zip/ZipIndex.cs similarity index 50% rename from src/ExcelReader.Core/Reader/Zip/ZipMemoryIndex.cs rename to src/ExcelReader.Core/Reader/Zip/ZipIndex.cs index 5ecb9d47..cb6b225d 100644 --- a/src/ExcelReader.Core/Reader/Zip/ZipMemoryIndex.cs +++ b/src/ExcelReader.Core/Reader/Zip/ZipIndex.cs @@ -3,6 +3,7 @@ using System.IO.Compression; using System.Runtime.InteropServices; using ExcelReader.Core.Reader.Internal; +using ExcelReader.Core.Reader.Sources; namespace ExcelReader.Core.Reader.Zip { @@ -40,7 +41,7 @@ public void Dispose() } } - internal sealed class ZipMemoryIndex : IDisposable + internal sealed class ZipIndex : IDisposable { private const int EocdFixedSize = 22; private const int Zip64LocatorSize = 20; @@ -57,50 +58,195 @@ internal sealed class ZipMemoryIndex : IDisposable private static ReadOnlySpan EocdSignatureBytes => [0x50, 0x4B, 0x05, 0x06]; + private const int TailWindowSize = 65535 + EocdFixedSize + Zip64LocatorSize; + + private readonly ByteSource _source; private readonly ReadOnlyMemory _file; + private readonly ReadOnlyMemory _directory; + private readonly byte[]? _directoryRented; private readonly ZipEntryRef[] _entries; - private bool _disposed; + private int _disposed; - private ZipMemoryIndex(ReadOnlyMemory file, ZipEntryRef[] entries, int count) + private ZipIndex(ByteSource source, ReadOnlyMemory directory, byte[]? directoryRented, ZipEntryRef[] entries, int count) { - _file = file; + _source = source; + HasMemory = source.TryGetMemory(out _file); + _directory = directory; + _directoryRented = directoryRented; _entries = entries; Count = count; } internal int Count { get; } - internal static ZipMemoryIndex Create(ReadOnlyMemory file, ExcelReaderOptions options) + internal bool HasMemory { get; } + + internal static ZipIndex Create(ReadOnlyMemory file, ExcelReaderOptions options) { - ReadOnlySpan span = file.Span; - long eocdOffset = FindEocd(span); - (long cdOffset, long cdSize, long declaredCount) = ReadEocdRecord(span, eocdOffset, out long zip64EocdOffset); - if (zip64EocdOffset >= 0) + return Create(ByteSource.FromMemory(file), options); + } + + internal static ZipIndex Create(ByteSource source, ExcelReaderOptions options) + { + byte[]? directoryRented = null; + try { - (cdOffset, cdSize, declaredCount) = ReadZip64Eocd(span, zip64EocdOffset); + long length = source.Length; + int tailSize = (int)Math.Min(length, TailWindowSize); + long cdOffset; + long cdSize; + long declaredCount; + long zip64EocdOffset; + if (source.TryGetMemory(out ReadOnlyMemory file)) + { + (cdOffset, cdSize, declaredCount, zip64EocdOffset) = ParseTail(file.Span[(file.Length - tailSize)..]); + } + else + { + byte[] tail = ArrayPool.Shared.Rent(Math.Max(1, tailSize)); + try + { + source.ReadExactly(length - tailSize, tail.AsSpan(0, tailSize)); + (cdOffset, cdSize, declaredCount, zip64EocdOffset) = ParseTail(tail.AsSpan(0, tailSize)); + } + finally + { + ArrayPool.Shared.Return(tail); + } + } + if (zip64EocdOffset >= 0) + { + ThrowIfZip64RecordOutOfRange(zip64EocdOffset, length); + Span record = stackalloc byte[Zip64EocdFixedSize]; + source.ReadExactly(zip64EocdOffset, record); + (cdOffset, cdSize, declaredCount) = ReadZip64Eocd(record, 0); + } + ThrowIfDirectoryOutOfRange(cdOffset, cdSize, length); + + ReadOnlyMemory directory; + if (source.TryGetMemory(out file)) + { + directory = file.Slice((int)cdOffset, (int)cdSize); + } + else + { + directoryRented = ArrayPool.Shared.Rent(Math.Max(1, (int)cdSize)); + source.ReadExactly(cdOffset, directoryRented.AsSpan(0, (int)cdSize)); + directory = directoryRented.AsMemory(0, (int)cdSize); + } + return Build(source, directory, directoryRented, declaredCount, options); } - if (cdOffset < 0 || cdSize < 0 || cdOffset > span.Length - cdSize) + catch + { + if (directoryRented is not null) + { + ArrayPool.Shared.Return(directoryRented); + } + source.Dispose(); + throw; + } + } + + internal static async ValueTask CreateAsync(ByteSource source, ExcelReaderOptions options, CancellationToken ct) + { + if (source.TryGetMemory(out _)) + { + return Create(source, options); + } + byte[]? directoryRented = null; + try + { + long length = source.Length; + int tailSize = (int)Math.Min(length, TailWindowSize); + long cdOffset; + long cdSize; + long declaredCount; + long zip64EocdOffset; + byte[] tail = ArrayPool.Shared.Rent(Math.Max(1, tailSize)); + try + { + await source.ReadExactlyAsync(length - tailSize, tail.AsMemory(0, tailSize), ct).ConfigureAwait(false); + (cdOffset, cdSize, declaredCount, zip64EocdOffset) = ParseTail(tail.AsSpan(0, tailSize)); + } + finally + { + ArrayPool.Shared.Return(tail); + } + if (zip64EocdOffset >= 0) + { + ThrowIfZip64RecordOutOfRange(zip64EocdOffset, length); + byte[] record = ArrayPool.Shared.Rent(Zip64EocdFixedSize); + try + { + await source.ReadExactlyAsync(zip64EocdOffset, record.AsMemory(0, Zip64EocdFixedSize), ct).ConfigureAwait(false); + (cdOffset, cdSize, declaredCount) = ReadZip64Eocd(record.AsSpan(0, Zip64EocdFixedSize), 0); + } + finally + { + ArrayPool.Shared.Return(record); + } + } + ThrowIfDirectoryOutOfRange(cdOffset, cdSize, length); + directoryRented = ArrayPool.Shared.Rent(Math.Max(1, (int)cdSize)); + await source.ReadExactlyAsync(cdOffset, directoryRented.AsMemory(0, (int)cdSize), ct).ConfigureAwait(false); + return Build(source, directoryRented.AsMemory(0, (int)cdSize), directoryRented, declaredCount, options); + } + catch + { + if (directoryRented is not null) + { + ArrayPool.Shared.Return(directoryRented); + } + source.Dispose(); + throw; + } + } + + private static (long CdOffset, long CdSize, long Count, long Zip64EocdOffset) ParseTail(ReadOnlySpan tail) + { + long eocdOffset = FindEocd(tail); + (long cdOffset, long cdSize, long declaredCount) = ReadEocdRecord(tail, eocdOffset, out long zip64EocdOffset); + return (cdOffset, cdSize, declaredCount, zip64EocdOffset); + } + + private static void ThrowIfZip64RecordOutOfRange(long offset, long length) + { + if (offset < 0 || offset > length - Zip64EocdFixedSize) + { + throw new InvalidDataException("The ZIP64 end of central directory record is out of range."); + } + } + + private static void ThrowIfDirectoryOutOfRange(long cdOffset, long cdSize, long length) + { + if (cdOffset < 0 || cdSize < 0 || cdOffset > length - cdSize) { throw new InvalidDataException("The ZIP central directory is out of range."); } + if (cdSize > Array.MaxLength) + { + throw new ExcelLimitExceededException("ArrayMaxLength", Array.MaxLength, cdSize); + } + } + private static ZipIndex Build(ByteSource source, ReadOnlyMemory directory, byte[]? directoryRented, long declaredCount, ExcelReaderOptions options) + { long maxHint = Math.Max(16, options.MaxZipEntries > 0 ? options.MaxZipEntries : 65_536); ZipEntryRef[] entries = ArrayPool.Shared.Rent((int)Math.Clamp(declaredCount, 16, maxHint)); - int count; try { - count = WalkCentralDirectory(span, cdOffset, cdSize, ref entries, options); - ThrowIfDuplicateEntryNames(file, entries, count); + int count = WalkCentralDirectory(directory.Span, ref entries, options); + ThrowIfDuplicateEntryNames(directory, entries, count); + return new ZipIndex(source, directory, directoryRented, entries, count); } catch { ArrayPool.Shared.Return(entries); throw; } - return new ZipMemoryIndex(file, entries, count); } - private static void ThrowIfDuplicateEntryNames(ReadOnlyMemory file, ZipEntryRef[] entries, int count) + private static void ThrowIfDuplicateEntryNames(ReadOnlyMemory directory, ZipEntryRef[] entries, int count) { if (count <= 1) { @@ -115,18 +261,18 @@ private static void ThrowIfDuplicateEntryNames(ReadOnlyMemory file, ZipEnt } Array.Sort(order, 0, count, Comparer.Create((a, b) => { - ReadOnlySpan span = file.Span; + ReadOnlySpan span = directory.Span; ref readonly ZipEntryRef ea = ref entries[a]; ref readonly ZipEntryRef eb = ref entries[b]; return span.Slice(ea.NameStart, ea.NameLength).SequenceCompareTo(span.Slice(eb.NameStart, eb.NameLength)); })); - ReadOnlySpan fileSpan = file.Span; + ReadOnlySpan directorySpan = directory.Span; for (int i = 1; i < count; i++) { ref readonly ZipEntryRef prev = ref entries[order[i - 1]]; ref readonly ZipEntryRef curr = ref entries[order[i]]; if (prev.NameLength == curr.NameLength && - fileSpan.Slice(prev.NameStart, prev.NameLength).SequenceEqual(fileSpan.Slice(curr.NameStart, curr.NameLength))) + directorySpan.Slice(prev.NameStart, prev.NameLength).SequenceEqual(directorySpan.Slice(curr.NameStart, curr.NameLength))) { throw new InvalidDataException("The ZIP central directory contains a duplicate entry name."); } @@ -138,12 +284,18 @@ private static void ThrowIfDuplicateEntryNames(ReadOnlyMemory file, ZipEnt } } + private void ThrowIfDisposed() + { + ObjectDisposedException.ThrowIf(Volatile.Read(ref _disposed) != 0, this); + } + internal bool TryGetEntry(ReadOnlySpan utf8Name, out ZipEntryRef entry) { - ReadOnlySpan fileSpan = _file.Span; + ThrowIfDisposed(); + ReadOnlySpan directory = _directory.Span; foreach (ref readonly ZipEntryRef candidate in _entries.AsSpan(0, Count)) { - if (fileSpan.Slice(candidate.NameStart, candidate.NameLength).SequenceEqual(utf8Name)) + if (directory.Slice(candidate.NameStart, candidate.NameLength).SequenceEqual(utf8Name)) { entry = candidate; return true; @@ -154,6 +306,106 @@ internal bool TryGetEntry(ReadOnlySpan utf8Name, out ZipEntryRef entry) } internal ZipPart OpenPart(in ZipEntryRef entry, DecompressedByteCounter counter, string entryLimitName = "", long entryLimit = 0) + { + ThrowIfDisposed(); + ThrowIfPartTooLarge(entry, counter, entryLimitName, entryLimit); + long dataOffset = ResolveDataOffset(entry); + ThrowIfEntryDataInvalid(dataOffset, entry); + ZipPart part; + if (HasMemory) + { + ReadOnlyMemory compressed = CompressedSlice(dataOffset, entry.CompressedSize); + part = entry.Method == 0 ? new ZipPart(compressed, rented: null) : InflateToPart(compressed, entry.UncompressedSize); + } + else + { + part = ReadPart(dataOffset, entry); + } + counter.Add(entry.UncompressedSize); + return part; + } + + internal async ValueTask OpenPartAsync( + ZipEntryRef entry, DecompressedByteCounter counter, CancellationToken ct, string entryLimitName = "", long entryLimit = 0) + { + ThrowIfDisposed(); + if (HasMemory) + { + return OpenPart(entry, counter, entryLimitName, entryLimit); + } + ThrowIfPartTooLarge(entry, counter, entryLimitName, entryLimit); + long dataOffset = await ResolveDataOffsetAsync(entry, ct).ConfigureAwait(false); + ThrowIfEntryDataInvalid(dataOffset, entry); + int size = (int)entry.UncompressedSize; + byte[] rented = ArrayPool.Shared.Rent(Math.Max(1, size)); + try + { + ByteSourceStream raw = new(_source, dataOffset, entry.CompressedSize); + Stream content = entry.Method == 0 ? raw : new DeflateStream(raw, CompressionMode.Decompress); + await using (content.ConfigureAwait(false)) + { + try + { + await content.ReadExactlyAsync(rented.AsMemory(0, size), ct).ConfigureAwait(false); + } + catch (EndOfStreamException ex) + { + throw new InvalidDataException("The ZIP entry produced less data than its declared uncompressed size.", ex); + } + if (await content.ReadAsync(new byte[1], ct).ConfigureAwait(false) != 0) + { + throw new InvalidDataException("The ZIP entry produced more data than its declared uncompressed size."); + } + } + } + catch + { + ArrayPool.Shared.Return(rented); + throw; + } + counter.Add(entry.UncompressedSize); + return new ZipPart(rented.AsMemory(0, size), rented); + } + + internal ValueTask OpenPartOrDefaultAsync(ReadOnlySpan utf8Name, DecompressedByteCounter counter, CancellationToken ct) + { + return TryGetEntry(utf8Name, out ZipEntryRef entry) ? OpenPartAsync(entry, counter, ct) : new ValueTask(default(ZipPart)); + } + + internal ZipPart OpenPartOrDefault(ReadOnlySpan utf8Name, DecompressedByteCounter counter) + { + return TryGetEntry(utf8Name, out ZipEntryRef entry) ? OpenPart(entry, counter) : default; + } + + internal LimitedReadStream OpenEntryStream( + in ZipEntryRef entry, DecompressedByteCounter counter, ExcelReaderOptions options, string entryLimitName = "", long entryLimit = 0) + { + ThrowIfDisposed(); + long dataOffset = ResolveDataOffset(entry); + return WrapEntryStream(entry, dataOffset, counter, options, entryLimitName, entryLimit); + } + + internal async ValueTask OpenEntryStreamAsync( + ZipEntryRef entry, DecompressedByteCounter counter, ExcelReaderOptions options, CancellationToken ct, + string entryLimitName = "", long entryLimit = 0) + { + ThrowIfDisposed(); + long dataOffset = HasMemory ? ResolveDataOffset(entry) : await ResolveDataOffsetAsync(entry, ct).ConfigureAwait(false); + return WrapEntryStream(entry, dataOffset, counter, options, entryLimitName, entryLimit); + } + + private LimitedReadStream WrapEntryStream( + in ZipEntryRef entry, long dataOffset, DecompressedByteCounter counter, ExcelReaderOptions options, string entryLimitName, long entryLimit) + { + ThrowIfEntryDataInvalid(dataOffset, entry); + Stream raw = HasMemory + ? ToReadableMemoryStream(CompressedSlice(dataOffset, entry.CompressedSize)) + : new ByteSourceStream(_source, dataOffset, entry.CompressedSize); + Stream opened = entry.Method == 0 ? raw : new DeflateStream(raw, CompressionMode.Decompress); + return WorkbookLookups.Wrap(opened, counter, options, entryLimitName, entryLimit, entry.UncompressedSize); + } + + private static void ThrowIfPartTooLarge(in ZipEntryRef entry, DecompressedByteCounter counter, string entryLimitName, long entryLimit) { LimitChecks.ThrowIfEntryLengthExceeds(entry.UncompressedSize, counter.Remaining, nameof(ExcelReaderOptions.MaxTotalDecompressedBytes)); if (entryLimit > 0) @@ -164,61 +416,151 @@ internal ZipPart OpenPart(in ZipEntryRef entry, DecompressedByteCounter counter, { throw new ExcelLimitExceededException("ArrayMaxLength", Array.MaxLength, entry.UncompressedSize); } + } - ReadOnlyMemory compressed = ResolveCompressedSlice(entry); - ZipPart part = entry.Method switch + private void ThrowIfEntryDataInvalid(long dataOffset, in ZipEntryRef entry) + { + long length = _source.Length; + if (dataOffset < 0 || dataOffset > length || entry.CompressedSize < 0 || entry.CompressedSize > length - dataOffset) { - 0 => new ZipPart(compressed, rented: null), - 8 => InflateToPart(compressed, entry.UncompressedSize), - _ => throw new NotSupportedException($"Unsupported ZIP compression method: {entry.Method}."), - }; - counter.Add(entry.UncompressedSize); - return part; + throw new InvalidDataException("The ZIP entry data runs past the end of the file."); + } + if (entry.Method is not (0 or 8)) + { + throw new NotSupportedException($"Unsupported ZIP compression method: {entry.Method}."); + } + if (entry.Method == 0 && entry.CompressedSize != entry.UncompressedSize) + { + throw new InvalidDataException("The stored ZIP entry's compressed and uncompressed sizes differ."); + } } - internal ZipPart OpenPartOrDefault(ReadOnlySpan utf8Name, DecompressedByteCounter counter) + private ReadOnlyMemory CompressedSlice(long dataOffset, long compressedSize) { - return TryGetEntry(utf8Name, out ZipEntryRef entry) ? OpenPart(entry, counter) : default; + if (compressedSize > Array.MaxLength) + { + throw new ExcelLimitExceededException("ArrayMaxLength", Array.MaxLength, compressedSize); + } + return _file.Slice((int)dataOffset, (int)compressedSize); } - internal Stream OpenEntryStream(in ZipEntryRef entry, DecompressedByteCounter counter, ExcelReaderOptions options, string entryLimitName = "", long entryLimit = 0) + private ZipPart ReadPart(long dataOffset, in ZipEntryRef entry) { - ReadOnlyMemory compressed = ResolveCompressedSlice(entry); - Stream opened = entry.Method switch + int size = (int)entry.UncompressedSize; + byte[] rented = ArrayPool.Shared.Rent(Math.Max(1, size)); + try { - 0 => ToReadableMemoryStream(compressed), - 8 => new DeflateStream(ToReadableMemoryStream(compressed), CompressionMode.Decompress), - _ => throw new NotSupportedException($"Unsupported ZIP compression method: {entry.Method}."), - }; - return WorkbookLookups.Wrap(opened, counter, options, entryLimitName, entryLimit, entry.UncompressedSize); + ByteSourceStream raw = new(_source, dataOffset, entry.CompressedSize); + using Stream content = entry.Method == 0 ? raw : new DeflateStream(raw, CompressionMode.Decompress); + try + { + content.ReadExactly(rented.AsSpan(0, size)); + } + catch (EndOfStreamException ex) + { + throw new InvalidDataException("The ZIP entry produced less data than its declared uncompressed size.", ex); + } + if (content.ReadByte() != -1) + { + throw new InvalidDataException("The ZIP entry produced more data than its declared uncompressed size."); + } + return new ZipPart(rented.AsMemory(0, size), rented); + } + catch + { + ArrayPool.Shared.Return(rented); + throw; + } } - private ReadOnlyMemory ResolveCompressedSlice(in ZipEntryRef entry) + private long ResolveDataOffset(in ZipEntryRef entry) { - long dataOffset = ResolveDataOffset(entry); - long fileLength = _file.Length; - if (dataOffset < 0 || dataOffset > fileLength || entry.CompressedSize < 0 || entry.CompressedSize > fileLength - dataOffset) + ThrowIfLocalHeaderOutOfRange(entry.LocalHeaderOffset); + Span header = stackalloc byte[LocalHeaderFixedSize]; + _source.ReadExactly(entry.LocalHeaderOffset, header); + long nameStart = ParseLocalHeader(header, entry, out int nameLength, out int extraLength); + byte[] name = ArrayPool.Shared.Rent(Math.Max(1, nameLength)); + try { - throw new InvalidDataException("The ZIP entry data runs past the end of the file."); + _source.ReadExactly(nameStart, name.AsSpan(0, nameLength)); + ThrowIfNameMismatch(name.AsSpan(0, nameLength), entry); } - if (entry.CompressedSize > Array.MaxLength) + finally { - throw new ExcelLimitExceededException("ArrayMaxLength", Array.MaxLength, entry.CompressedSize); + ArrayPool.Shared.Return(name); + } + return nameStart + nameLength + extraLength; + } + + private async ValueTask ResolveDataOffsetAsync(ZipEntryRef entry, CancellationToken ct) + { + ThrowIfLocalHeaderOutOfRange(entry.LocalHeaderOffset); + byte[] buffer = ArrayPool.Shared.Rent(LocalHeaderFixedSize + ushort.MaxValue); + try + { + await _source.ReadExactlyAsync(entry.LocalHeaderOffset, buffer.AsMemory(0, LocalHeaderFixedSize), ct).ConfigureAwait(false); + long nameStart = ParseLocalHeader(buffer.AsSpan(0, LocalHeaderFixedSize), entry, out int nameLength, out int extraLength); + await _source.ReadExactlyAsync(nameStart, buffer.AsMemory(LocalHeaderFixedSize, nameLength), ct).ConfigureAwait(false); + ThrowIfNameMismatch(buffer.AsSpan(LocalHeaderFixedSize, nameLength), entry); + return nameStart + nameLength + extraLength; + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + private void ThrowIfLocalHeaderOutOfRange(long headerOffset) + { + if (headerOffset < 0 || headerOffset > _source.Length - LocalHeaderFixedSize) + { + throw new InvalidDataException("The ZIP local file header is out of range."); + } + } + + private long ParseLocalHeader(ReadOnlySpan header, in ZipEntryRef entry, out int nameLength, out int extraLength) + { + if (BinaryPrimitives.ReadInt32LittleEndian(header) != LocalFileHeaderSignature) + { + throw new InvalidDataException("Invalid ZIP local file header signature."); + } + nameLength = BinaryPrimitives.ReadUInt16LittleEndian(header[26..]); + extraLength = BinaryPrimitives.ReadUInt16LittleEndian(header[28..]); + long nameStart = entry.LocalHeaderOffset + LocalHeaderFixedSize; + if (nameLength > _source.Length - nameStart) + { + throw new InvalidDataException("The ZIP local file header name runs past the end of the file."); + } + if (nameLength != entry.NameLength) + { + throw new InvalidDataException("The ZIP local file header name does not match the central directory."); + } + return nameStart; + } + + private void ThrowIfNameMismatch(ReadOnlySpan localName, in ZipEntryRef entry) + { + if (!localName.SequenceEqual(_directory.Span.Slice(entry.NameStart, entry.NameLength))) + { + throw new InvalidDataException("The ZIP local file header name does not match the central directory."); } - return _file.Slice((int)dataOffset, (int)entry.CompressedSize); } public void Dispose() { - if (_disposed) + if (Interlocked.Exchange(ref _disposed, 1) != 0) { return; } - _disposed = true; if (_entries.Length > 0) { ArrayPool.Shared.Return(_entries); } + if (_directoryRented is not null) + { + ArrayPool.Shared.Return(_directoryRented); + } + _source.Dispose(); } private static long FindEocd(ReadOnlySpan span) @@ -244,7 +586,7 @@ private static long FindEocd(ReadOnlySpan span) continue; } int commentLength = BinaryPrimitives.ReadUInt16LittleEndian(window.Slice(index + 20, 2)); - if (index + EocdFixedSize + commentLength == window.Length) + if (index + EocdFixedSize + commentLength <= window.Length) { return windowStart + index; } @@ -298,11 +640,10 @@ private static (long CdOffset, long CdSize, long Count) ReadZip64Eocd(ReadOnlySp return (cdOffset, cdSize, count); } - private static int WalkCentralDirectory(ReadOnlySpan span, long cdOffset, long cdSize, ref ZipEntryRef[] entries, ExcelReaderOptions options) + private static int WalkCentralDirectory(ReadOnlySpan directory, ref ZipEntryRef[] entries, ExcelReaderOptions options) { - // Create has checked cdOffset <= span.Length - cdSize, so end <= span.Length and cannot overflow. - long end = cdOffset + cdSize; - long pos = cdOffset; + int end = directory.Length; + int pos = 0; int count = 0; while (pos < end) { @@ -310,7 +651,7 @@ private static int WalkCentralDirectory(ReadOnlySpan span, long cdOffset, { throw new InvalidDataException("The ZIP central directory is truncated."); } - ReadOnlySpan record = span.Slice((int)pos, CentralDirectoryFixedSize); + ReadOnlySpan record = directory.Slice(pos, CentralDirectoryFixedSize); if (BinaryPrimitives.ReadInt32LittleEndian(record) != CentralDirectorySignature) { throw new InvalidDataException("Invalid ZIP central directory record signature."); @@ -321,21 +662,20 @@ private static int WalkCentralDirectory(ReadOnlySpan span, long cdOffset, throw new NotSupportedException("Encrypted ZIP entries are not supported."); } - long nameStart = pos + CentralDirectoryFixedSize; + int nameStart = pos + CentralDirectoryFixedSize; int variableLength = fields.NameLength + fields.ExtraLength + fields.CommentLength; if (variableLength > end - nameStart) { throw new InvalidDataException("The ZIP central directory is truncated."); } - long recordEnd = nameStart + variableLength; (long compressed, long uncompressed, long localOffset) = ResolveZip64Sizes( - span, (int)(nameStart + fields.NameLength), fields.ExtraLength, fields); + directory, nameStart + fields.NameLength, fields.ExtraLength, fields); EnsureCapacity(ref entries, count); entries[count] = new ZipEntryRef { - NameStart = (int)nameStart, + NameStart = nameStart, NameLength = fields.NameLength, LocalHeaderOffset = localOffset, CompressedSize = compressed, @@ -345,7 +685,7 @@ private static int WalkCentralDirectory(ReadOnlySpan span, long cdOffset, }; count++; LimitChecks.ThrowIfTooManyEntries(count, options); - pos = recordEnd; + pos = nameStart + variableLength; } return count; } @@ -448,34 +788,6 @@ private static void EnsureCapacity(ref ZipEntryRef[] entries, int count) entries = bigger; } - private long ResolveDataOffset(in ZipEntryRef entry) - { - ReadOnlySpan fileSpan = _file.Span; - long headerOffset = entry.LocalHeaderOffset; - if (headerOffset < 0 || headerOffset > fileSpan.Length - LocalHeaderFixedSize) - { - throw new InvalidDataException("The ZIP local file header is out of range."); - } - ReadOnlySpan header = fileSpan.Slice((int)headerOffset, LocalHeaderFixedSize); - if (BinaryPrimitives.ReadInt32LittleEndian(header) != LocalFileHeaderSignature) - { - throw new InvalidDataException("Invalid ZIP local file header signature."); - } - ushort nameLength = BinaryPrimitives.ReadUInt16LittleEndian(header[26..]); - ushort extraLength = BinaryPrimitives.ReadUInt16LittleEndian(header[28..]); - long nameStart = headerOffset + LocalHeaderFixedSize; - if (nameLength > fileSpan.Length - nameStart) - { - throw new InvalidDataException("The ZIP local file header name runs past the end of the file."); - } - if (nameLength != entry.NameLength || - !fileSpan.Slice((int)nameStart, nameLength).SequenceEqual(fileSpan.Slice(entry.NameStart, entry.NameLength))) - { - throw new InvalidDataException("The ZIP local file header name does not match the central directory."); - } - return nameStart + nameLength + extraLength; - } - private static ZipPart InflateToPart(ReadOnlyMemory compressed, long uncompressedSize) { int size = checked((int)uncompressedSize); diff --git a/src/ExcelReader.Core/Reader/Zip/ZipReaderOpen.cs b/src/ExcelReader.Core/Reader/Zip/ZipReaderOpen.cs deleted file mode 100644 index a9430d9e..00000000 --- a/src/ExcelReader.Core/Reader/Zip/ZipReaderOpen.cs +++ /dev/null @@ -1,81 +0,0 @@ -using System.IO.Compression; -using ExcelReader.Core.Reader.Internal; - -namespace ExcelReader.Core.Reader.Zip -{ - internal static class ZipReaderOpen - { - internal static ZipArchive Open(Stream stream, bool leaveOpen) - { - try - { - return new ZipArchive(stream, ZipArchiveMode.Read, leaveOpen: true); - } - catch - { - if (!leaveOpen) - { - stream.Dispose(); - } - throw; - } - } - - internal static async ValueTask OpenAsync( - Stream stream, bool leaveOpen, ExcelReaderOptions options, Func> parseBody, CancellationToken ct = default) - { - ZipArchive? zip = null; - try - { - zip = await ZipArchive.CreateAsync(stream, ZipArchiveMode.Read, leaveOpen: true, entryNameEncoding: null, ct).ConfigureAwait(false); - LimitChecks.ThrowIfTooManyEntries(zip.Entries.Count, options); - return await parseBody(zip).ConfigureAwait(false); - } - catch - { - if (zip is not null) - { - await zip.DisposeAsync().ConfigureAwait(false); - } - if (!leaveOpen) - { - await stream.DisposeAsync().ConfigureAwait(false); - } - throw; - } - } - - internal static async ValueTask FromOpenZipAsync( - Stream stream, bool leaveOpen, ZipArchive zip, ExcelReaderOptions options, - Func> parseBody) - { - try - { - LimitChecks.ThrowIfTooManyEntries(zip.Entries.Count, options); - return await parseBody(zip).ConfigureAwait(false); - } - catch - { - await zip.DisposeAsync().ConfigureAwait(false); - if (!leaveOpen) - { - await stream.DisposeAsync().ConfigureAwait(false); - } - throw; - } - } - - internal static TResult FromMemory(ZipMemoryIndex memZip, Func build) - { - try - { - return build(memZip); - } - catch - { - memZip.Dispose(); - throw; - } - } - } -} diff --git a/src/ExcelReader.Core/Writer/Xls/BiffRecord.cs b/src/ExcelReader.Core/Writer/Xls/BiffRecord.cs index 47e78a12..53f15481 100644 --- a/src/ExcelReader.Core/Writer/Xls/BiffRecord.cs +++ b/src/ExcelReader.Core/Writer/Xls/BiffRecord.cs @@ -27,6 +27,7 @@ internal static class BiffRecord internal const int BookBool = 0x00DA; internal const int Dimension = 0x0200; internal const int Window2 = 0x023E; + internal const int Blank = 0x0201; internal const int Number = 0x0203; internal const int Label = 0x0204; internal const int BoolErr = 0x0205; diff --git a/src/ExcelReader.Core/Writer/Xls/BiffRecordWriter.cs b/src/ExcelReader.Core/Writer/Xls/BiffRecordWriter.cs index 053c54c9..1b205a00 100644 --- a/src/ExcelReader.Core/Writer/Xls/BiffRecordWriter.cs +++ b/src/ExcelReader.Core/Writer/Xls/BiffRecordWriter.cs @@ -218,6 +218,13 @@ internal static void WriteWindow2(BiffBuffer buffer) buffer.EndRecord(len); } + internal static void WriteBlank(BiffBuffer buffer, int row, int col, int xf) + { + int len = buffer.BeginRecord(BiffRecord.Blank); + WriteCellHeader(buffer, row, col, xf); + buffer.EndRecord(len); + } + internal static void WriteNumber(BiffBuffer buffer, int row, int col, int xf, double value) { CellValueGuards.ThrowIfNonFinite(value, nameof(value)); diff --git a/src/ExcelReader.Core/Writer/Xls/XlsSheetWriter.cs b/src/ExcelReader.Core/Writer/Xls/XlsSheetWriter.cs index 7bb20635..86517f61 100644 --- a/src/ExcelReader.Core/Writer/Xls/XlsSheetWriter.cs +++ b/src/ExcelReader.Core/Writer/Xls/XlsSheetWriter.cs @@ -31,6 +31,7 @@ public sealed class XlsSheetWriter : ISheetWriter private int _rowNumber = -1; private WriterState _state = WriterState.Created; private bool _rowActive; + private bool _rowHasCells; private Dictionary? _columnStyles; private Dictionary? _columnWidths; private int _activeRowStyle; @@ -158,13 +159,20 @@ public XlsRowWriter StartRow(int styleId) } _activeRowStyle = styleId; _rowActive = true; + _rowHasCells = false; _rowWriter ??= new XlsRowWriter(this, _rowNumber); _rowWriter.Reset(_rowNumber); return _rowWriter; } + /// A row left without cells gets one blank cell, since BIFF8 has no other trace of it for a reader to find. internal void NotifyRowEnded() { + if (_rowActive && !_rowHasCells) + { + BiffRecordWriter.WriteBlank(_cells, _rowNumber, 0, XlsGlobals.GeneralXf); + Track(_rowNumber, 0); + } _rowActive = false; } @@ -209,6 +217,7 @@ internal void ReleaseBuffer() private void Track(int row, int col) { + _rowHasCells = true; if (row > _maxRow) { _maxRow = row; } if (col > _maxCol) { _maxCol = col; } } diff --git a/tests/ExcelReader.Tests/Arrow/ArrowWriteExtensionsTests.cs b/tests/ExcelReader.Tests/Arrow/ArrowWriteExtensionsTests.cs index d52c5d4f..084d49f1 100644 --- a/tests/ExcelReader.Tests/Arrow/ArrowWriteExtensionsTests.cs +++ b/tests/ExcelReader.Tests/Arrow/ArrowWriteExtensionsTests.cs @@ -214,13 +214,147 @@ public void WriteRecordBatch_WithHeaderFalse_OmitsTheHeaderRow() Assert.False(e.MoveNext()); } + private static RecordBatch BuildWideBatch() + { + var decimalType = new Decimal128Type(10, 2); + var secondTime = new Time32Type(TimeUnit.Second); + + var i8 = new Int8Array.Builder(); + i8.Append(-8); + i8.Append(7); + var i16 = new Int16Array.Builder(); + i16.Append(-16); + i16.AppendNull(); + var i32 = new Int32Array.Builder(); + i32.Append(-32); + i32.AppendNull(); + var u8 = new UInt8Array.Builder(); + u8.Append(8); + u8.AppendNull(); + var u16 = new UInt16Array.Builder(); + u16.Append(16); + u16.AppendNull(); + var u32 = new UInt32Array.Builder(); + u32.Append(32); + u32.AppendNull(); + var u64 = new UInt64Array.Builder(); + u64.Append(64); + u64.AppendNull(); + var half = new HalfFloatArray.Builder(); + half.Append((Half)0.5f); + half.AppendNull(); + var single = new FloatArray.Builder(); + single.Append(0.1f); + single.AppendNull(); + var dec = new Decimal128Array.Builder(decimalType); + dec.Append(12.34m); + dec.AppendNull(); + var view = new StringViewArray.Builder(); + view.Append("alice"); + view.AppendNull(); + var date = new Date64Array.Builder(); + date.Append(new DateOnly(2024, 1, 15)); + date.AppendNull(); + var time = new Time32Array.Builder(secondTime); + time.Append(new TimeOnly(13, 30, 0)); + time.AppendNull(); + + Schema schema = new Schema.Builder() + .Field(new Field("i8", Int8Type.Default, nullable: true)) + .Field(new Field("i16", Int16Type.Default, nullable: true)) + .Field(new Field("i32", Int32Type.Default, nullable: true)) + .Field(new Field("u8", UInt8Type.Default, nullable: true)) + .Field(new Field("u16", UInt16Type.Default, nullable: true)) + .Field(new Field("u32", UInt32Type.Default, nullable: true)) + .Field(new Field("u64", UInt64Type.Default, nullable: true)) + .Field(new Field("half", HalfFloatType.Default, nullable: true)) + .Field(new Field("single", FloatType.Default, nullable: true)) + .Field(new Field("dec", decimalType, nullable: true)) + .Field(new Field("view", StringViewType.Default, nullable: true)) + .Field(new Field("date", Date64Type.Default, nullable: true)) + .Field(new Field("time", secondTime, nullable: true)) + .Field(new Field("nothing", NullType.Default, nullable: true)) + .Build(); + + return new RecordBatch( + schema, + [ + i8.Build(), i16.Build(), i32.Build(), u8.Build(), u16.Build(), u32.Build(), u64.Build(), + half.Build(), single.Build(), dec.Build(), view.Build(), date.Build(), time.Build(), new NullArray(2), + ], + 2); + } + + private static void AssertWideRoundTrips(IExcelRowReader reader) + { + ExcelColumnType[] types = + [ + ExcelColumnType.Int64Column, ExcelColumnType.Int64Column, ExcelColumnType.Int64Column, + ExcelColumnType.Int64Column, ExcelColumnType.Int64Column, ExcelColumnType.Int64Column, + ExcelColumnType.Int64Column, ExcelColumnType.Float64Column, ExcelColumnType.Float64Column, + ExcelColumnType.Float64Column, ExcelColumnType.StringColumn, ExcelColumnType.DateColumn, + ExcelColumnType.TimeColumn, ExcelColumnType.StringColumn, + ]; + var schema = new ExcelColumnSchema[types.Length]; + for (int i = 0; i < types.Length; i++) + { + schema[i] = new() { Index = i, Name = "c" + i, Type = types[i], IsNullable = true }; + } + RecordBatch batch = reader.ToArrowRecordBatch(schema); + + Assert.Equal(2, batch.Length); + long[] integers = [-8, -16, -32, 8, 16, 32, 64]; + for (int i = 0; i < integers.Length; i++) + { + Assert.Equal(integers[i], Assert.IsType(batch.Column(i)).GetValue(0)); + } + Assert.Equal(0.5, Assert.IsType(batch.Column(7)).GetValue(0)); + Assert.Equal(0.1, Assert.IsType(batch.Column(8)).GetValue(0)); + Assert.Equal(12.34, Assert.IsType(batch.Column(9)).GetValue(0)); + Assert.Equal("alice", Assert.IsType(batch.Column(10)).GetString(0)); + Assert.Equal(new DateOnly(2024, 1, 15), Assert.IsType(batch.Column(11)).GetDateOnly(0)); + Assert.Equal(new TimeOnly(13, 30, 0), Assert.IsType(batch.Column(12)).GetTime(0)); + Assert.Equal(7L, ((Int64Array)batch.Column(0)).GetValue(1)); + for (int i = 1; i < types.Length; i++) + { + Assert.True(batch.Column(i).IsNull(1), "column " + i); + } + Assert.True(batch.Column(13).IsNull(0)); + } + + [Fact] + public void WriteRecordBatch_Xlsx_RoundTrips_Widened_Types() + { + using var ms = new MemoryStream(); + using (XlsxWorkbookWriter workbook = XlsxWorkbookWriter.Create(ms, leaveOpen: true)) + { + workbook.WriteRecordBatch(BuildWideBatch()); + } + + using XlsxReader reader = Excel.FromXlsx(ms.ToArray()); + AssertWideRoundTrips(reader); + } + + [Fact] + public void WriteRecordBatch_Xlsb_RoundTrips_Widened_Types() + { + using var ms = new MemoryStream(); + using (XlsbWorkbookWriter workbook = XlsbWorkbookWriter.Create(ms, leaveOpen: true)) + { + workbook.WriteRecordBatch(BuildWideBatch()); + } + + using var reader = Excel.FromXlsb(ms.ToArray()); + AssertWideRoundTrips(reader); + } + [Fact] public void WriteRecordBatch_UnsupportedArrowType_Throws() { - var col = new FloatArray.Builder(); - col.Append(1.5f); + var col = new BinaryArray.Builder(); + col.Append(new byte[] { 1 }.AsSpan()); Schema schema = new Schema.Builder() - .Field(new Field("f", FloatType.Default, nullable: false)) + .Field(new Field("f", BinaryType.Default, nullable: false)) .Build(); var batch = new RecordBatch(schema, [col.Build()], 1); diff --git a/tests/ExcelReader.Tests/Reader/ConcurrentSheetTests.cs b/tests/ExcelReader.Tests/Reader/ConcurrentSheetTests.cs new file mode 100644 index 00000000..3682dafa --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/ConcurrentSheetTests.cs @@ -0,0 +1,268 @@ +using System.Globalization; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Writer; +using ExcelReader.Core.Writer.Xls; +using ExcelReader.Core.Writer.Xlsb; +using ExcelReader.Core.Writer.Xlsx; + +namespace ExcelReader.Tests.Reader +{ + public sealed class ConcurrentSheetTests : IDisposable + { + public enum Format + { + Xlsx, + Xlsb, + Xls, + } + + public enum Source + { + Memory, + Stream, + Path, + } + + private const int Sheets = 6; + private const int Rows = 3000; + + private readonly List _paths = []; + + public void Dispose() + { + foreach (string path in _paths) + { + File.Delete(path); + } + } + + public static TheoryData Cases + { + get + { + TheoryData data = []; + foreach (Format format in Enum.GetValues()) + { + foreach (Source source in Enum.GetValues()) + { + data.Add(format, source); + } + } + return data; + } + } + + private static void Fill(IWorkbookWriter workbook) + where TSheet : ISheetWriter + where TRow : IRowWriter + { + for (int s = 0; s < Sheets; s++) + { + TSheet sheet = workbook.AddSheet("sheet" + s.ToString(CultureInfo.InvariantCulture)); + for (int r = 0; r < Rows; r++) + { + using TRow row = sheet.StartRow(); + row.Write("s" + s.ToString(CultureInfo.InvariantCulture) + "-name-" + (r % 200).ToString(CultureInfo.InvariantCulture)); + row.Write((s * 1_000_000) + r); + row.Write("shared-" + (r % 17).ToString(CultureInfo.InvariantCulture)); + } + sheet.End(); + sheet.Dispose(); + } + workbook.End(); + } + + private static byte[] Build(Format format) + { + using MemoryStream buffer = new(); + switch (format) + { + case Format.Xlsx: + using (XlsxWorkbookWriter xlsx = XlsxWorkbookWriter.Create(buffer, leaveOpen: true, new XlsxWriterOptions { UseSharedStrings = true })) + { + Fill(xlsx); + } + break; + case Format.Xlsb: + using (XlsbWorkbookWriter xlsb = XlsbWorkbookWriter.Create(buffer, leaveOpen: true, new XlsbWriterOptions { UseSharedStrings = true })) + { + Fill(xlsb); + } + break; + default: + using (XlsWorkbookWriter xls = XlsWorkbookWriter.Create(buffer, leaveOpen: true)) + { + Fill(xls); + } + break; + } + return buffer.ToArray(); + } + + private IExcelRowReader Open(Format format, Source source, byte[] bytes) + { + if (source == Source.Memory) + { + return Excel.Open(bytes); + } + if (source == Source.Stream) + { + return Excel.Open(new MemoryStream(bytes, writable: false), leaveOpen: false); + } + string path = Path.Combine(Path.GetTempPath(), "excelreader-concurrent-" + Guid.NewGuid().ToString("N") + "." + format.ToString()); + File.WriteAllBytes(path, bytes); + _paths.Add(path); + return Excel.Open(path); + } + + private static List Drain(IExcelRowEnumerator e) + { + List lines = []; + while (e.MoveNext()) + { + Row row = e.Current; + lines.Add(row[0].GetString() + "|" + row[1].GetString() + "|" + row[2].GetString()); + } + return lines; + } + + private static List[] ReadSequentially(IExcelRowReader reader) + { + List[] sheets = new List[reader.SheetCount]; + for (int s = 0; s < sheets.Length; s++) + { + reader.MoveToSheet(s); + using IExcelRowEnumerator e = reader.GetEnumerator(); + sheets[s] = Drain(e); + } + return sheets; + } + + [Theory] + [MemberData(nameof(Cases))] + public void Sheets_Read_In_Parallel_Match_A_Sequential_Read(Format format, Source source) + { + byte[] bytes = Build(format); + List[] expected; + using (IExcelRowReader reference = Excel.Open(bytes)) + { + expected = ReadSequentially(reference); + } + Assert.Equal(Sheets, expected.Length); + Assert.All(expected, sheet => Assert.Equal(Rows, sheet.Count)); + + using IExcelRowReader reader = Open(format, source, bytes); + IExcelRowEnumerator[] enumerators = new IExcelRowEnumerator[Sheets]; + for (int s = 0; s < Sheets; s++) + { + reader.MoveToSheet(s); + enumerators[s] = reader.GetEnumerator(); + } + + List[] actual = new List[Sheets]; + Parallel.For(0, Sheets, new ParallelOptions { MaxDegreeOfParallelism = Sheets }, s => + { + using IExcelRowEnumerator e = enumerators[s]; + actual[s] = Drain(e); + }); + + for (int s = 0; s < Sheets; s++) + { + Assert.Equal(expected[s], actual[s]); + } + } + + [Theory] + [MemberData(nameof(Cases))] + public async Task Sheets_Read_In_Parallel_Asynchronously_Match_A_Sequential_Read(Format format, Source source) + { + CancellationToken ct = TestContext.Current.CancellationToken; + byte[] bytes = Build(format); + List[] expected; + using (IExcelRowReader reference = Excel.Open(bytes)) + { + expected = ReadSequentially(reference); + } + + using IExcelRowReader reader = Open(format, source, bytes); + IExcelRowEnumerator[] enumerators = new IExcelRowEnumerator[Sheets]; + for (int s = 0; s < Sheets; s++) + { + reader.MoveToSheet(s); + enumerators[s] = reader.GetAsyncEnumerator(ct); + } + + Task>[] tasks = new Task>[Sheets]; + for (int s = 0; s < Sheets; s++) + { + IExcelRowEnumerator e = enumerators[s]; + tasks[s] = Task.Run( + async () => + { + List lines = []; + await using (e) + { + while (await e.MoveNextAsync()) + { + Row row = e.Current; + lines.Add(row[0].GetString() + "|" + row[1].GetString() + "|" + row[2].GetString()); + } + } + return lines; + }, + ct); + } + List[] actual = await Task.WhenAll(tasks); + + for (int s = 0; s < Sheets; s++) + { + Assert.Equal(expected[s], actual[s]); + } + } + + [Theory] + [MemberData(nameof(Cases))] + public void Threads_Reading_One_Sheet_Get_The_Same_String_Instances(Format format, Source source) + { + byte[] bytes = Build(format); + using IExcelRowReader reader = Open(format, source, bytes); + reader.MoveToSheet(0); + + const int Threads = 8; + IExcelRowEnumerator[] enumerators = new IExcelRowEnumerator[Threads]; + for (int t = 0; t < Threads; t++) + { + enumerators[t] = reader.GetEnumerator(); + } + + string[][] thirdColumn = new string[Threads][]; + Parallel.For(0, Threads, new ParallelOptions { MaxDegreeOfParallelism = Threads }, t => + { + using IExcelRowEnumerator e = enumerators[t]; + string[] values = new string[Rows]; + int r = 0; + while (e.MoveNext()) + { + values[r++] = e.Current[2].GetString(); + } + thirdColumn[t] = values; + }); + + for (int r = 0; r < Rows; r++) + { + Assert.Equal("shared-" + (r % 17).ToString(CultureInfo.InvariantCulture), thirdColumn[0][r]); + for (int t = 1; t < Threads; t++) + { + if (format == Format.Xls) + { + Assert.Equal(thirdColumn[0][r], thirdColumn[t][r]); + } + else + { + Assert.Same(thirdColumn[0][r], thirdColumn[t][r]); + } + } + } + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Csv/CsvLifetimeTests.cs b/tests/ExcelReader.Tests/Reader/Csv/CsvLifetimeTests.cs new file mode 100644 index 00000000..fce632ef --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Csv/CsvLifetimeTests.cs @@ -0,0 +1,233 @@ +using System.Globalization; +using System.Text; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Csv; + +namespace ExcelReader.Tests.Reader.Csv +{ + public sealed class CsvLifetimeTests : IDisposable + { + private const int Rows = 4000; + + private readonly List _paths = []; + + public void Dispose() + { + foreach (string path in _paths) + { + File.Delete(path); + } + } + + private static byte[] BuildCsv(Encoding? encoding = null) + { + StringBuilder text = new(); + for (int r = 0; r < Rows; r++) + { + text.Append("name-").Append(r.ToString(CultureInfo.InvariantCulture)) + .Append(',').Append((r * 7).ToString(CultureInfo.InvariantCulture)) + .Append(",\"quoted, ").Append((r % 13).ToString(CultureInfo.InvariantCulture)).Append("\"\n"); + } + return (encoding ?? Encoding.UTF8).GetBytes(text.ToString()); + } + + private static List Drain(CsvReader.Enumerator e) + { + List lines = []; + while (e.MoveNext()) + { + Row row = e.Current; + lines.Add(row[0].GetString() + "|" + row[1].GetString() + "|" + row[2].GetString()); + } + return lines; + } + + private static List Expected() + { + using CsvReader reference = Excel.FromCsv(BuildCsv()); + using CsvReader.Enumerator all = reference.GetEnumerator(); + return Drain(all); + } + + [Fact] + public void Two_Enumerators_Over_One_Seekable_Stream_Do_Not_Share_A_Position() + { + List expected = Expected(); + using CsvReader reader = Excel.FromCsv(new TrickleStream(BuildCsv()), leaveOpen: false); + using CsvReader.Enumerator first = reader.GetEnumerator(); + using CsvReader.Enumerator second = reader.GetEnumerator(); + + List a = []; + List b = []; + bool moreA = true; + bool moreB = true; + while (moreA || moreB) + { + moreA = moreA && first.MoveNext(); + if (moreA) + { + a.Add(first.Current[0].GetString() + "|" + first.Current[1].GetString() + "|" + first.Current[2].GetString()); + } + moreB = moreB && second.MoveNext(); + if (moreB) + { + b.Add(second.Current[0].GetString() + "|" + second.Current[1].GetString() + "|" + second.Current[2].GetString()); + } + } + + Assert.Equal(expected, a, StringComparer.Ordinal); + Assert.Equal(expected, b, StringComparer.Ordinal); + } + + [Theory] + [InlineData("memory")] + [InlineData("stream")] + [InlineData("path")] + public void Enumerators_On_Different_Threads_Each_Read_Every_Row(string source) + { + List expected = Expected(); + byte[] bytes = BuildCsv(); + using CsvReader reader = Open(source, bytes); + + const int Threads = 6; + CsvReader.Enumerator[] enumerators = new CsvReader.Enumerator[Threads]; + for (int t = 0; t < Threads; t++) + { + enumerators[t] = reader.GetEnumerator(); + } + List[] actual = new List[Threads]; + Parallel.For(0, Threads, new ParallelOptions { MaxDegreeOfParallelism = Threads }, t => + { + using CsvReader.Enumerator e = enumerators[t]; + actual[t] = Drain(e); + }); + + Assert.All(actual, lines => Assert.Equal(expected, lines, StringComparer.Ordinal)); + } + + private CsvReader Open(string source, byte[] bytes) + { + if (string.Equals(source, "memory", StringComparison.Ordinal)) + { + return Excel.FromCsv(bytes); + } + if (string.Equals(source, "stream", StringComparison.Ordinal)) + { + return Excel.FromCsv(new TrickleStream(bytes), leaveOpen: false); + } + string path = Path.Combine(Path.GetTempPath(), "excelreader-csv-" + Guid.NewGuid().ToString("N") + ".csv"); + File.WriteAllBytes(path, bytes); + _paths.Add(path); + return Excel.FromCsvFile(path); + } + + [Fact] + public void A_Transcoded_Stream_Serves_Two_Enumerators() + { + List expected = Expected(); + using CsvReader reader = Excel.FromCsv( + new TrickleStream(BuildCsv(Encoding.Unicode)), leaveOpen: false, + new CsvReaderOptions { Encoding = Encoding.Unicode, DetectEncodingFromByteOrderMark = false }); + using CsvReader.Enumerator first = reader.GetEnumerator(); + using CsvReader.Enumerator second = reader.GetEnumerator(); + Assert.Equal(expected, Drain(first), StringComparer.Ordinal); + Assert.Equal(expected, Drain(second), StringComparer.Ordinal); + } + + [Fact] + public void A_Stream_Already_Advanced_By_The_Caller_Starts_Every_Enumerator_There() + { + List expected = Expected(); + byte[] csv = BuildCsv(); + byte[] prefix = Encoding.UTF8.GetBytes("this line is not part of the data\n"); + TrickleStream stream = new([.. prefix, .. csv]); + stream.Position = prefix.Length; + + using CsvReader reader = Excel.FromCsv(stream, leaveOpen: false); + using CsvReader.Enumerator first = reader.GetEnumerator(); + using CsvReader.Enumerator second = reader.GetEnumerator(); + Assert.Equal(expected, Drain(first), StringComparer.Ordinal); + Assert.Equal(expected, Drain(second), StringComparer.Ordinal); + } + + [Fact] + public void A_NonSeekable_Stream_Serves_Exactly_One_Enumerator() + { + using NonSeekableStream stream = new(BuildCsv()); + using CsvReader reader = Excel.FromCsv(stream, leaveOpen: true); + + int granted = 0; + int refused = 0; + CsvReader.Enumerator?[] taken = new CsvReader.Enumerator?[8]; + Parallel.For(0, taken.Length, new ParallelOptions { MaxDegreeOfParallelism = taken.Length }, t => + { + try + { + taken[t] = reader.GetEnumerator(); + Interlocked.Increment(ref granted); + } + catch (InvalidOperationException) + { + Interlocked.Increment(ref refused); + } + }); + + Assert.Equal(1, granted); + Assert.Equal(taken.Length - 1, refused); + using CsvReader.Enumerator only = taken.Single(e => e is not null)!; + Assert.Equal(Expected(), Drain(only), StringComparer.Ordinal); + } + + [Fact] + public void An_Enumerator_Outlives_Its_Reader() + { + List expected = Expected(); + TrickleStream stream = new(BuildCsv()); + CsvReader reader = Excel.FromCsv(stream, leaveOpen: false); + CsvReader.Enumerator e = reader.GetEnumerator(); + Assert.True(e.MoveNext()); + + reader.Dispose(); + Assert.True(stream.CanRead); + Assert.Equal(expected.Skip(1), Drain(e), StringComparer.Ordinal); + + e.Dispose(); + Assert.False(stream.CanRead); + } + + [Fact] + public void GetEnumerator_After_Dispose_Throws() + { + CsvReader reader = Excel.FromCsv(new TrickleStream(BuildCsv()), leaveOpen: false); + reader.Dispose(); + Assert.Throws(() => reader.GetEnumerator()); + Assert.Throws(() => reader.GetAsyncEnumerator(TestContext.Current.CancellationToken)); + } + + [Fact] + public void A_Transcoded_File_Offers_No_Raw_Chunk_Source() + { + string path = Path.Combine(Path.GetTempPath(), "excelreader-csv-" + Guid.NewGuid().ToString("N") + ".csv"); + File.WriteAllBytes(path, BuildCsv(Encoding.Unicode)); + _paths.Add(path); + + using CsvReader utf16 = Excel.FromCsvFile(path, new CsvReaderOptions { Encoding = Encoding.Unicode, DetectEncodingFromByteOrderMark = false }); + Assert.False(utf16.TryGetChunkSource(out _)); + + using CsvReader utf8 = Excel.FromCsvFile(path); + Assert.True(utf8.TryGetChunkSource(out _)); + } + + [Fact] + public void A_Borrowed_Stream_Is_Left_Open() + { + TrickleStream stream = new(BuildCsv()); + using (CsvReader reader = Excel.FromCsv(stream, leaveOpen: true)) + { + using CsvReader.Enumerator e = reader.GetEnumerator(); + Assert.True(e.MoveNext()); + } + Assert.True(stream.CanRead); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Csv/CsvStructuralScannerTests.cs b/tests/ExcelReader.Tests/Reader/Csv/CsvStructuralScannerTests.cs new file mode 100644 index 00000000..a485a550 --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Csv/CsvStructuralScannerTests.cs @@ -0,0 +1,23 @@ +using ExcelReader.Core.Reader.Csv; + +namespace ExcelReader.Tests.Reader.Csv +{ + public sealed class CsvStructuralScannerTests + { + [Fact] + public void PrefixXor_Matches_The_Software_Ladder() + { + ulong[] edges = [0, 1, 1UL << 63, ulong.MaxValue, 0x5555_5555_5555_5555, 0x8000_0000_0000_0001]; + foreach (ulong edge in edges) + { + Assert.Equal(CsvStructuralScanner.PrefixXorSoftware(edge), CsvStructuralScanner.PrefixXor(edge)); + } + ulong bits = 1; + for (int i = 0; i < 10_000; i++) + { + bits = (bits * 6364136223846793005UL) + 1442695040888963407UL; + Assert.Equal(CsvStructuralScanner.PrefixXorSoftware(bits), CsvStructuralScanner.PrefixXor(bits)); + } + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/EmptyRowParityTests.cs b/tests/ExcelReader.Tests/Reader/EmptyRowParityTests.cs new file mode 100644 index 00000000..4f1a172d --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/EmptyRowParityTests.cs @@ -0,0 +1,113 @@ +using ExcelReader.Core.Reader; +using ExcelReader.Core.Writer; +using ExcelReader.Core.Writer.Xls; +using ExcelReader.Core.Writer.Xlsb; +using ExcelReader.Core.Writer.Xlsx; +using ExcelReader.Tests.Reader.Xls; + +namespace ExcelReader.Tests.Reader +{ + public sealed class EmptyRowParityTests + { + private const int RowRecord = 0x0208; + private const int LabelRecord = 0x0204; + + private static void WriteSparseSheet(IWorkbookWriter workbook) + where TSheet : ISheetWriter + where TRow : IRowWriter + { + TSheet sheet = workbook.AddSheet("s"); + string?[] values = ["a", null, "b", null]; + foreach (string? value in values) + { + using TRow row = sheet.StartRow(); + row.Write(value); + row.Write((long?)null); + } + sheet.End(); + sheet.Dispose(); + workbook.End(); + } + + private static List FirstCells(TEnumerator e) + where TEnumerator : IExcelRowEnumerator + { + using TEnumerator owned = e; + var cells = new List(); + while (owned.MoveNext()) + { + cells.Add(owned.Current.ColumnCount == 0 ? null : owned.Current[0].GetString()); + } + return cells; + } + + [Fact] + public void Xlsx_Yields_Rows_Written_Without_Cells() + { + using var ms = new MemoryStream(); + using (XlsxWorkbookWriter workbook = XlsxWorkbookWriter.Create(ms, leaveOpen: true)) + { + WriteSparseSheet(workbook); + } + + using var reader = Excel.FromXlsx(ms.ToArray()); + Assert.Equal(["a", null, "b", null], FirstCells(reader.GetEnumerator())); + } + + [Fact] + public void Xlsb_Yields_Rows_Written_Without_Cells() + { + using var ms = new MemoryStream(); + using (XlsbWorkbookWriter workbook = XlsbWorkbookWriter.Create(ms, leaveOpen: true)) + { + WriteSparseSheet(workbook); + } + + using var reader = Excel.FromXlsb(ms.ToArray()); + Assert.Equal(["a", null, "b", null], FirstCells(reader.GetEnumerator())); + } + + [Fact] + public void Xls_Yields_Rows_Written_Without_Cells() + { + using var ms = new MemoryStream(); + using (XlsWorkbookWriter workbook = XlsWorkbookWriter.Create(ms, leaveOpen: true)) + { + WriteSparseSheet(workbook); + } + + using var reader = Excel.FromXls(ms.ToArray()); + Assert.Equal(["a", null, "b", null], FirstCells(reader.GetEnumerator())); + } + + [Fact] + public void Xls_Yields_A_Row_Record_Without_Cells_In_Row_Order() + { + using var ms = XlsWorkbookBuilder.BuildRawSheet( + includeEof: true, + (RowRecord, XlsWorkbookBuilder.RawRowOnly(0)), + (RowRecord, XlsWorkbookBuilder.RawRowOnly(1)), + (RowRecord, XlsWorkbookBuilder.RawRowOnly(2)), + (RowRecord, XlsWorkbookBuilder.RawRowOnly(3)), + (LabelRecord, XlsWorkbookBuilder.RawLabel(0, 0, "a")), + (LabelRecord, XlsWorkbookBuilder.RawLabel(2, 0, "b"))); + + using var reader = Excel.FromXls(ms); + Assert.Equal(["a", null, "b", null], FirstCells(reader.GetEnumerator())); + } + + [Fact] + public void Xls_Does_Not_Repeat_A_Row_Whose_Row_Record_Follows_Its_Cells() + { + using var ms = XlsWorkbookBuilder.BuildRawSheet( + includeEof: true, + (LabelRecord, XlsWorkbookBuilder.RawLabel(0, 0, "a")), + (LabelRecord, XlsWorkbookBuilder.RawLabel(1, 0, "b")), + (RowRecord, XlsWorkbookBuilder.RawRowOnly(0)), + (RowRecord, XlsWorkbookBuilder.RawRowOnly(1))); + + using var reader = Excel.FromXls(ms); + Assert.Equal(["a", "b"], FirstCells(reader.GetEnumerator())); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/ExcelDataReaderTests.cs b/tests/ExcelReader.Tests/Reader/ExcelDataReaderTests.cs index 6fef2f24..c8bf18c3 100644 --- a/tests/ExcelReader.Tests/Reader/ExcelDataReaderTests.cs +++ b/tests/ExcelReader.Tests/Reader/ExcelDataReaderTests.cs @@ -3,7 +3,9 @@ using System.Text; using ExcelReader.Core.Reader; using ExcelReader.Core.Reader.Xlsb; +using ExcelReader.Core.Reader.Xlsx; using ExcelReader.Core.Writer.Xlsb; +using ExcelReader.Core.Writer.Xlsx; namespace ExcelReader.Tests.Reader { @@ -327,5 +329,34 @@ public async Task DataTableLoadPopulatesFromTheSheet() Assert.Equal(2.0, Convert.ToDouble(table.Rows[1]["Id"], CultureInfo.InvariantCulture)); Assert.Equal("bob", table.Rows[1]["Name"]); } + + [Fact] + public void AFailedConstructorDoesNotKeepTheReaderAlive() + { + using MemoryStream buffer = new(); + using (XlsxWorkbookWriter workbook = XlsxWorkbookWriter.Create(buffer, leaveOpen: true)) + { + XlsxSheetWriter sheet = workbook.AddSheet("s"); + using (XlsxRowWriter row = sheet.StartRow()) + { + for (int c = 0; c < 200; c++) + { + row.Write(c); + } + } + sheet.End(); + sheet.Dispose(); + workbook.End(); + } + + TrickleStream stream = new(buffer.ToArray()); + XlsxReader reader = Excel.FromXlsx(stream, leaveOpen: false, new ExcelReaderOptions { MaxCellBytes = 64 }); + Assert.Throws(() => + { + using ExcelDataReader data = new(reader); + }); + reader.Dispose(); + Assert.False(stream.CanRead); + } } } diff --git a/tests/ExcelReader.Tests/Reader/Internal/DecompressedByteCounterTests.cs b/tests/ExcelReader.Tests/Reader/Internal/DecompressedByteCounterTests.cs new file mode 100644 index 00000000..82252468 --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Internal/DecompressedByteCounterTests.cs @@ -0,0 +1,44 @@ +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Zip; + +namespace ExcelReader.Tests.Reader.Internal +{ + public sealed class DecompressedByteCounterTests + { + [Fact] + public void Concurrent_Adds_Are_All_Counted() + { + const int Threads = 16; + const int PerThread = 20_000; + DecompressedByteCounter counter = new(Threads * PerThread); + + Parallel.For(0, Threads, new ParallelOptions { MaxDegreeOfParallelism = Threads }, _ => + { + for (int i = 0; i < PerThread; i++) + { + counter.Add(1); + } + }); + + Assert.Equal(0, counter.Remaining); + Assert.Throws(() => counter.Add(1)); + } + + [Fact] + public void Passing_The_Limit_On_One_Thread_Fails_The_Others_Too() + { + DecompressedByteCounter counter = new(100); + Assert.Throws(() => counter.Add(101)); + Assert.Throws(() => counter.Add(1)); + } + + [Fact] + public void A_Counter_Without_A_Limit_Never_Throws() + { + DecompressedByteCounter counter = new(0); + counter.Add(long.MaxValue); + counter.Add(long.MaxValue); + Assert.Equal(long.MaxValue, counter.Remaining); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Internal/OnceGateTests.cs b/tests/ExcelReader.Tests/Reader/Internal/OnceGateTests.cs new file mode 100644 index 00000000..de50b555 --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Internal/OnceGateTests.cs @@ -0,0 +1,111 @@ +using ExcelReader.Core.Reader.Internal; + +namespace ExcelReader.Tests.Reader.Internal +{ + public sealed class OnceGateTests + { + private sealed class Probe + { + internal int Loads; + internal int Resets; + internal Exception? Fail; + } + + private static void Load(Probe probe) + { + Interlocked.Increment(ref probe.Loads); + if (probe.Fail is not null) + { + throw probe.Fail; + } + } + + private static void Reset(Probe probe) + { + Interlocked.Increment(ref probe.Resets); + } + + [Fact] + public void Load_Runs_Once_For_Many_Concurrent_Callers() + { + using OnceGate gate = new(); + Probe probe = new(); + Parallel.For(0, 64, _ => gate.Run(probe, Load, Reset)); + Assert.Equal(1, probe.Loads); + Assert.True(gate.IsDone); + } + + [Fact] + public void A_Failure_Is_Rethrown_Without_Loading_Again() + { + using OnceGate gate = new(); + Probe probe = new() { Fail = new InvalidDataException("bad table") }; + + InvalidDataException first = Assert.Throws(() => gate.Run(probe, Load, Reset)); + probe.Fail = null; + InvalidDataException second = Assert.Throws(() => gate.Run(probe, Load, Reset)); + + Assert.Same(first, second); + Assert.Equal(1, probe.Loads); + Assert.Equal(0, probe.Resets); + Assert.False(gate.IsDone); + } + + [Fact] + public void Cancellation_Resets_And_Lets_The_Next_Caller_Load() + { + using OnceGate gate = new(); + Probe probe = new() { Fail = new OperationCanceledException() }; + + Assert.Throws(() => gate.Run(probe, Load, Reset)); + Assert.Equal(1, probe.Resets); + Assert.False(gate.IsDone); + + probe.Fail = null; + gate.Run(probe, Load, Reset); + Assert.Equal(2, probe.Loads); + Assert.True(gate.IsDone); + } + + [Fact] + public async Task RunAsync_Follows_The_Same_Rules() + { + CancellationToken ct = TestContext.Current.CancellationToken; + using OnceGate gate = new(); + Probe probe = new() { Fail = new OperationCanceledException() }; + + static ValueTask LoadAsync(Probe p, CancellationToken token) + { + Load(p); + return ValueTask.CompletedTask; + } + + await Assert.ThrowsAsync(async () => await gate.RunAsync(probe, LoadAsync, Reset, ct)); + Assert.Equal(1, probe.Resets); + + probe.Fail = new InvalidDataException("bad table"); + await Assert.ThrowsAsync(async () => await gate.RunAsync(probe, LoadAsync, Reset, ct)); + probe.Fail = null; + await Assert.ThrowsAsync(async () => await gate.RunAsync(probe, LoadAsync, Reset, ct)); + Assert.Equal(2, probe.Loads); + } + + [Fact] + public async Task A_Sync_Caller_And_An_Async_Caller_Share_One_Load() + { + CancellationToken ct = TestContext.Current.CancellationToken; + using OnceGate gate = new(); + Probe probe = new(); + + static ValueTask LoadAsync(Probe p, CancellationToken token) + { + Load(p); + return ValueTask.CompletedTask; + } + + await gate.RunAsync(probe, LoadAsync, Reset, ct); + gate.Run(probe, Load, Reset); + Assert.Equal(1, probe.Loads); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Internal/ReaderLifetimeTests.cs b/tests/ExcelReader.Tests/Reader/Internal/ReaderLifetimeTests.cs new file mode 100644 index 00000000..ce3e685b --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Internal/ReaderLifetimeTests.cs @@ -0,0 +1,67 @@ +using ExcelReader.Core.Reader.Internal; + +namespace ExcelReader.Tests.Reader.Internal +{ + public sealed class ReaderLifetimeTests + { + [Fact] + public void Close_Alone_Releases() + { + int released = 0; + ReaderLifetime lifetime = new(() => released++); + lifetime.Close(); + Assert.Equal(1, released); + } + + [Fact] + public void Release_Happens_When_The_Last_Holder_Lets_Go() + { + int released = 0; + ReaderLifetime lifetime = new(() => released++); + lifetime.Acquire(this); + lifetime.Acquire(this); + + lifetime.Close(); + Assert.Equal(0, released); + lifetime.Release(); + Assert.Equal(0, released); + lifetime.Release(); + Assert.Equal(1, released); + } + + [Fact] + public void Close_Twice_Releases_Once() + { + int released = 0; + ReaderLifetime lifetime = new(() => released++); + lifetime.Close(); + lifetime.Close(); + Assert.Equal(1, released); + } + + [Fact] + public void Acquire_After_Close_Throws_Even_While_Holders_Remain() + { + ReaderLifetime lifetime = new(() => { }); + lifetime.Acquire(this); + lifetime.Close(); + ObjectDisposedException ex = Assert.Throws(() => lifetime.Acquire(this)); + Assert.Contains(nameof(ReaderLifetimeTests), ex.ObjectName, StringComparison.Ordinal); + } + + [Fact] + public void Concurrent_Holders_Release_Exactly_Once() + { + int released = 0; + ReaderLifetime lifetime = new(() => Interlocked.Increment(ref released)); + Parallel.For(0, 64, _ => + { + lifetime.Acquire(this); + lifetime.Release(); + }); + Assert.Equal(0, released); + lifetime.Close(); + Assert.Equal(1, released); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Internal/SharedStringCacheTests.cs b/tests/ExcelReader.Tests/Reader/Internal/SharedStringCacheTests.cs new file mode 100644 index 00000000..9c3fe56e --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Internal/SharedStringCacheTests.cs @@ -0,0 +1,62 @@ +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Xlsx; +using ExcelReader.Core.Writer.Xlsx; + +namespace ExcelReader.Tests.Reader.Internal +{ + public sealed class SharedStringCacheTests + { + private static byte[] BuildXlsx() + { + using MemoryStream buffer = new(); + using (XlsxWorkbookWriter workbook = XlsxWorkbookWriter.Create(buffer, leaveOpen: true, new XlsxWriterOptions { UseSharedStrings = true })) + { + XlsxSheetWriter sheet = workbook.AddSheet("s"); + for (int r = 0; r < 50; r++) + { + using XlsxRowWriter row = sheet.StartRow(); + row.Write("value-" + (r % 5).ToString(System.Globalization.CultureInfo.InvariantCulture)); + } + sheet.End(); + sheet.Dispose(); + workbook.End(); + } + return buffer.ToArray(); + } + + private static List FirstColumn(XlsxReader reader) + { + List values = []; + using XlsxReader.Enumerator e = reader.GetEnumerator(); + while (e.MoveNext()) + { + values.Add(e.Current[0].GetString()); + } + return values; + } + + [Fact] + public void A_Shared_String_Is_Materialized_Once_Per_Reader() + { + using XlsxReader reader = Excel.FromXlsx(BuildXlsx()); + List first = FirstColumn(reader); + List second = FirstColumn(reader); + + Assert.Equal(50, first.Count); + for (int i = 0; i < first.Count; i++) + { + Assert.Same(first[i], second[i]); + Assert.Same(first[i % 5], first[i]); + } + } + + [Fact] + public void The_Cache_Array_Is_One_Instance() + { + using XlsxReader reader = Excel.FromXlsx(BuildXlsx()); + _ = FirstColumn(reader); + string?[] seen = reader.SharedStringCache; + Parallel.For(0, 32, _ => Assert.Same(seen, reader.SharedStringCache)); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/SourceParityTests.cs b/tests/ExcelReader.Tests/Reader/SourceParityTests.cs new file mode 100644 index 00000000..db336e24 --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/SourceParityTests.cs @@ -0,0 +1,537 @@ +using System.Globalization; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Xls; +using ExcelReader.Core.Reader.Xlsb; +using ExcelReader.Core.Reader.Xlsx; +using ExcelReader.Core.Writer.Xls; +using ExcelReader.Core.Writer.Xlsb; +using ExcelReader.Core.Writer.Xlsx; + +namespace ExcelReader.Tests.Reader +{ + public sealed class SourceParityTests : IDisposable + { + public enum Kind + { + Memory, + ExposedMemoryStream, + OpaqueMemoryStream, + Trickle, + FileStream, + FilePath, + NonSeekable, + } + + private readonly List _paths = []; + + public void Dispose() + { + foreach (string path in _paths) + { + File.Delete(path); + } + } + + public static TheoryData Kinds => + [ + Kind.Memory, Kind.ExposedMemoryStream, Kind.OpaqueMemoryStream, Kind.Trickle, + Kind.FileStream, Kind.FilePath, Kind.NonSeekable, + ]; + + private static byte[] BuildXlsx() + { + using MemoryStream buffer = new(); + using (XlsxWorkbookWriter workbook = XlsxWorkbookWriter.Create(buffer, leaveOpen: true, new XlsxWriterOptions { UseSharedStrings = true })) + { + for (int s = 0; s < 3; s++) + { + XlsxSheetWriter sheet = workbook.AddSheet("sheet" + s); + for (int r = 0; r < 400; r++) + { + using XlsxRowWriter row = sheet.StartRow(); + row.Write("name-" + ((s * 1000) + (r % 50)).ToString(CultureInfo.InvariantCulture)); + row.Write((s * 100000) + r); + row.Write(r % 2 == 0); + } + sheet.End(); + sheet.Dispose(); + } + workbook.End(); + } + return buffer.ToArray(); + } + + private string WriteTemp(byte[] bytes, string extension) + { + string path = Path.Combine(Path.GetTempPath(), "excelreader-parity-" + Guid.NewGuid().ToString("N") + extension); + File.WriteAllBytes(path, bytes); + _paths.Add(path); + return path; + } + + private XlsxReader OpenXlsx(Kind kind, byte[] bytes) + { + switch (kind) + { + case Kind.Memory: + return Excel.FromXlsx(bytes); + case Kind.ExposedMemoryStream: + MemoryStream exposed = new(); + exposed.Write(bytes); + exposed.Position = 0; + return Excel.FromXlsx(exposed, leaveOpen: false); + case Kind.OpaqueMemoryStream: + return Excel.FromXlsx(new MemoryStream(bytes, writable: false), leaveOpen: false); + case Kind.Trickle: + return Excel.FromXlsx(new TrickleStream(bytes), leaveOpen: false); + case Kind.FileStream: + return Excel.FromXlsx(File.OpenRead(WriteTemp(bytes, ".xlsx")), leaveOpen: false); + case Kind.FilePath: + return Excel.FromXlsxFile(WriteTemp(bytes, ".xlsx")); + default: + return Excel.FromXlsx(new NonSeekableStream(bytes), leaveOpen: false); + } + } + + private static List Dump(IExcelRowReader reader) + { + List lines = []; + for (int s = 0; s < reader.SheetCount; s++) + { + reader.MoveToSheet(s); + using IExcelRowEnumerator e = reader.GetEnumerator(); + while (e.MoveNext()) + { + lines.Add(Line(s, e.Current)); + } + } + return lines; + } + + private static async Task> DumpAsync(IExcelRowReader reader) + { + List lines = []; + for (int s = 0; s < reader.SheetCount; s++) + { + reader.MoveToSheet(s); + await using IExcelRowEnumerator e = reader.GetAsyncEnumerator(TestContext.Current.CancellationToken); + while (await e.MoveNextAsync()) + { + lines.Add(Line(s, e.Current)); + } + } + return lines; + } + + private static string Line(int sheet, Row row) + { + string[] cells = new string[row.ColumnCount]; + for (int c = 0; c < cells.Length; c++) + { + cells[c] = row[c].GetString(); + } + return sheet.ToString(CultureInfo.InvariantCulture) + "|" + string.Join("|", cells); + } + + [Theory] + [MemberData(nameof(Kinds))] + public void Xlsx_Reads_The_Same_Rows_From_Every_Source(Kind kind) + { + byte[] bytes = BuildXlsx(); + using IExcelRowReader expected = Excel.FromXlsx(bytes); + using IExcelRowReader actual = OpenXlsx(kind, bytes); + Assert.Equal(Dump(expected), Dump(actual)); + } + + [Theory] + [MemberData(nameof(Kinds))] + public async Task Xlsx_Reads_The_Same_Rows_Asynchronously_From_Every_Source(Kind kind) + { + byte[] bytes = BuildXlsx(); + using IExcelRowReader expected = Excel.FromXlsx(bytes); + using IExcelRowReader actual = OpenXlsx(kind, bytes); + Assert.Equal(Dump(expected), await DumpAsync(actual)); + } + + [Theory] + [MemberData(nameof(Kinds))] + public void Xlsx_Interleaved_Enumerators_Keep_Their_Own_Position(Kind kind) + { + byte[] bytes = BuildXlsx(); + using IExcelRowReader reader = OpenXlsx(kind, bytes); + reader.MoveToSheet(0); + using IExcelRowEnumerator first = reader.GetEnumerator(); + reader.MoveToSheet(1); + using IExcelRowEnumerator second = reader.GetEnumerator(); + for (int r = 0; r < 400; r++) + { + Assert.True(first.MoveNext()); + Assert.True(second.MoveNext()); + Assert.Equal("name-" + (r % 50).ToString(CultureInfo.InvariantCulture), first.Current[0].GetString()); + Assert.Equal("name-" + (1000 + (r % 50)).ToString(CultureInfo.InvariantCulture), second.Current[0].GetString()); + } + Assert.False(first.MoveNext()); + Assert.False(second.MoveNext()); + } + + [Fact] + public void Xlsx_File_Can_Be_Deleted_After_Dispose() + { + string path = WriteTemp(BuildXlsx(), ".xlsx"); + using (IExcelRowReader reader = Excel.FromXlsxFile(path)) + { + Assert.NotEmpty(Dump(reader)); + } + File.Delete(path); + Assert.False(File.Exists(path)); + } + + [Fact] + public void Xlsx_Leaves_A_Borrowed_Stream_Usable() + { + byte[] bytes = BuildXlsx(); + using TrickleStream stream = new(bytes); + using (IExcelRowReader reader = Excel.FromXlsx(stream, leaveOpen: true)) + { + Assert.NotEmpty(Dump(reader)); + } + Assert.True(stream.CanRead); + } + + [Fact] + public void Xlsx_Cut_Short_Is_Invalid_Data() + { + byte[] bytes = BuildXlsx(); + string path = WriteTemp(bytes[..(bytes.Length / 2)], ".xlsx"); + Assert.Throws(() => Excel.FromXlsxFile(path)); + } + + private static byte[] BuildXls() + { + using MemoryStream buffer = new(); + using (XlsWorkbookWriter workbook = XlsWorkbookWriter.Create(buffer, leaveOpen: true)) + { + for (int s = 0; s < 3; s++) + { + XlsSheetWriter sheet = workbook.AddSheet("sheet" + s); + for (int r = 0; r < 400; r++) + { + using XlsRowWriter row = sheet.StartRow(); + row.Write("name-" + ((s * 1000) + (r % 50)).ToString(CultureInfo.InvariantCulture)); + row.Write((s * 100000) + r); + row.Write(r % 2 == 0); + } + sheet.End(); + sheet.Dispose(); + } + workbook.End(); + } + return buffer.ToArray(); + } + + private XlsReader OpenXls(Kind kind, byte[] bytes) + { + switch (kind) + { + case Kind.Memory: + return Excel.FromXls(bytes); + case Kind.ExposedMemoryStream: + MemoryStream exposed = new(); + exposed.Write(bytes); + exposed.Position = 0; + return Excel.FromXls(exposed, leaveOpen: false); + case Kind.OpaqueMemoryStream: + return Excel.FromXls(new MemoryStream(bytes, writable: false), leaveOpen: false); + case Kind.Trickle: + return Excel.FromXls(new TrickleStream(bytes), leaveOpen: false); + case Kind.FileStream: + return Excel.FromXls(File.OpenRead(WriteTemp(bytes, ".xls")), leaveOpen: false); + case Kind.FilePath: + return Excel.FromXlsFile(WriteTemp(bytes, ".xls")); + default: + return Excel.FromXls(new NonSeekableStream(bytes), leaveOpen: false); + } + } + + [Theory] + [MemberData(nameof(Kinds))] + public void Xls_Reads_The_Same_Rows_From_Every_Source(Kind kind) + { + byte[] bytes = BuildXls(); + using IExcelRowReader expected = Excel.FromXls(bytes); + using IExcelRowReader actual = OpenXls(kind, bytes); + Assert.Equal(Dump(expected), Dump(actual)); + } + + [Theory] + [MemberData(nameof(Kinds))] + public void Xls_Interleaved_Enumerators_Keep_Their_Own_Position(Kind kind) + { + byte[] bytes = BuildXls(); + using IExcelRowReader reader = OpenXls(kind, bytes); + reader.MoveToSheet(0); + using IExcelRowEnumerator first = reader.GetEnumerator(); + reader.MoveToSheet(1); + using IExcelRowEnumerator second = reader.GetEnumerator(); + for (int r = 0; r < 400; r++) + { + Assert.True(first.MoveNext()); + Assert.True(second.MoveNext()); + Assert.Equal("name-" + (r % 50).ToString(CultureInfo.InvariantCulture), first.Current[0].GetString()); + Assert.Equal("name-" + (1000 + (r % 50)).ToString(CultureInfo.InvariantCulture), second.Current[0].GetString()); + } + Assert.False(first.MoveNext()); + Assert.False(second.MoveNext()); + } + + [Fact] + public void Xls_File_Can_Be_Deleted_After_Dispose() + { + string path = WriteTemp(BuildXls(), ".xls"); + using (IExcelRowReader reader = Excel.FromXlsFile(path)) + { + Assert.NotEmpty(Dump(reader)); + } + File.Delete(path); + Assert.False(File.Exists(path)); + } + + private static byte[] BuildXlsb() + { + using MemoryStream buffer = new(); + using (XlsbWorkbookWriter workbook = XlsbWorkbookWriter.Create(buffer, leaveOpen: true, new XlsbWriterOptions { UseSharedStrings = true })) + { + for (int s = 0; s < 3; s++) + { + XlsbSheetWriter sheet = workbook.AddSheet("sheet" + s); + for (int r = 0; r < 400; r++) + { + using XlsbRowWriter row = sheet.StartRow(); + row.Write("name-" + ((s * 1000) + (r % 50)).ToString(CultureInfo.InvariantCulture)); + row.Write((s * 100000) + r); + row.Write(r % 2 == 0); + } + sheet.End(); + sheet.Dispose(); + } + workbook.End(); + } + return buffer.ToArray(); + } + + private XlsbReader OpenXlsb(Kind kind, byte[] bytes) + { + switch (kind) + { + case Kind.Memory: + return Excel.FromXlsb(bytes); + case Kind.ExposedMemoryStream: + MemoryStream exposed = new(); + exposed.Write(bytes); + exposed.Position = 0; + return Excel.FromXlsb(exposed, leaveOpen: false); + case Kind.OpaqueMemoryStream: + return Excel.FromXlsb(new MemoryStream(bytes, writable: false), leaveOpen: false); + case Kind.Trickle: + return Excel.FromXlsb(new TrickleStream(bytes), leaveOpen: false); + case Kind.FileStream: + return Excel.FromXlsb(File.OpenRead(WriteTemp(bytes, ".xlsb")), leaveOpen: false); + case Kind.FilePath: + return Excel.FromXlsbFile(WriteTemp(bytes, ".xlsb")); + default: + return Excel.FromXlsb(new NonSeekableStream(bytes), leaveOpen: false); + } + } + + [Theory] + [MemberData(nameof(Kinds))] + public void Xlsb_Reads_The_Same_Rows_From_Every_Source(Kind kind) + { + byte[] bytes = BuildXlsb(); + using IExcelRowReader expected = Excel.FromXlsb(bytes); + using IExcelRowReader actual = OpenXlsb(kind, bytes); + Assert.Equal(Dump(expected), Dump(actual)); + } + + [Theory] + [MemberData(nameof(Kinds))] + public async Task Xlsb_Reads_The_Same_Rows_Asynchronously_From_Every_Source(Kind kind) + { + byte[] bytes = BuildXlsb(); + using IExcelRowReader expected = Excel.FromXlsb(bytes); + using IExcelRowReader actual = OpenXlsb(kind, bytes); + Assert.Equal(Dump(expected), await DumpAsync(actual)); + } + + [Fact] + public void Xlsb_File_Can_Be_Deleted_After_Dispose() + { + string path = WriteTemp(BuildXlsb(), ".xlsb"); + using (IExcelRowReader reader = Excel.FromXlsbFile(path)) + { + Assert.NotEmpty(Dump(reader)); + } + File.Delete(path); + Assert.False(File.Exists(path)); + } + + [Fact] + public void Xlsb_Cut_Short_Is_Invalid_Data() + { + byte[] bytes = BuildXlsb(); + string path = WriteTemp(bytes[..(bytes.Length / 2)], ".xlsb"); + Assert.Throws(() => Excel.FromXlsbFile(path)); + } + + public static TheoryData SeekableStreamKinds => + [ + Kind.ExposedMemoryStream, Kind.OpaqueMemoryStream, Kind.Trickle, Kind.FileStream, + ]; + + private Stream OpenRawStream(Kind kind, byte[] bytes) + { + switch (kind) + { + case Kind.ExposedMemoryStream: + MemoryStream exposed = new(); + exposed.Write(bytes); + exposed.Position = 0; + return exposed; + case Kind.OpaqueMemoryStream: + return new MemoryStream(bytes, writable: false); + case Kind.Trickle: + return new TrickleStream(bytes); + default: + return File.OpenRead(WriteTemp(bytes, ".bin")); + } + } + + [Theory] + [MemberData(nameof(SeekableStreamKinds))] + public void DetectFileFormat_Leaves_The_Stream_Open_At_Its_Position(Kind kind) + { + using Stream xlsx = OpenRawStream(kind, BuildXlsx()); + Assert.Equal(ExcelFileFormat.Xlsx, Excel.DetectFileFormat(xlsx)); + Assert.Equal(0, xlsx.Position); + Assert.True(xlsx.CanRead); + + using Stream xlsb = OpenRawStream(kind, BuildXlsb()); + Assert.Equal(ExcelFileFormat.Xlsb, Excel.DetectFileFormat(xlsb)); + Assert.Equal(0, xlsb.Position); + Assert.True(xlsb.CanRead); + } + + [Theory] + [MemberData(nameof(SeekableStreamKinds))] + public async Task DetectFileFormatAsync_Leaves_The_Stream_Open_At_Its_Position(Kind kind) + { + using Stream xlsx = OpenRawStream(kind, BuildXlsx()); + Assert.Equal(ExcelFileFormat.Xlsx, await Excel.DetectFileFormatAsync(xlsx, TestContext.Current.CancellationToken)); + Assert.Equal(0, xlsx.Position); + Assert.True(xlsx.CanRead); + } + + [Theory] + [MemberData(nameof(SeekableStreamKinds))] + public void Open_Detects_And_Reads_Both_Zip_Formats(Kind kind) + { + byte[] xlsx = BuildXlsx(); + using IExcelRowReader expectedXlsx = Excel.FromXlsx(xlsx); + using IExcelRowReader actualXlsx = Excel.Open(OpenRawStream(kind, xlsx), leaveOpen: false); + Assert.Equal(Dump(expectedXlsx), Dump(actualXlsx)); + + byte[] xlsb = BuildXlsb(); + using IExcelRowReader expectedXlsb = Excel.FromXlsb(xlsb); + using IExcelRowReader actualXlsb = Excel.Open(OpenRawStream(kind, xlsb), leaveOpen: false); + Assert.Equal(Dump(expectedXlsb), Dump(actualXlsb)); + } + + [Theory] + [MemberData(nameof(SeekableStreamKinds))] + public async Task OpenAsync_Detects_And_Reads_Both_Zip_Formats(Kind kind) + { + CancellationToken ct = TestContext.Current.CancellationToken; + byte[] xlsx = BuildXlsx(); + using IExcelRowReader expectedXlsx = Excel.FromXlsx(xlsx); + using IExcelRowReader actualXlsx = await Excel.OpenAsync(OpenRawStream(kind, xlsx), leaveOpen: false, ct: ct); + Assert.Equal(Dump(expectedXlsx), await DumpAsync(actualXlsx)); + + byte[] xlsb = BuildXlsb(); + using IExcelRowReader expectedXlsb = Excel.FromXlsb(xlsb); + using IExcelRowReader actualXlsb = await Excel.OpenAsync(OpenRawStream(kind, xlsb), leaveOpen: false, ct: ct); + Assert.Equal(Dump(expectedXlsb), await DumpAsync(actualXlsb)); + } + + private static byte[] CorruptZip() + { + byte[] bytes = new byte[200]; + bytes.AsSpan().Fill(0x41); + bytes[0] = 0x50; + bytes[1] = 0x4B; + bytes[2] = 0x03; + bytes[3] = 0x04; + return bytes; + } + + private static byte[] ZipWithoutWorkbook() + { + using MemoryStream buffer = new(); + using (System.IO.Compression.ZipArchive zip = new(buffer, System.IO.Compression.ZipArchiveMode.Create, leaveOpen: true)) + { + using StreamWriter writer = new(zip.CreateEntry("hello.txt").Open()); + writer.Write("hello"); + } + return buffer.ToArray(); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task A_Corrupt_Zip_Opened_Through_Every_Entry_Point_Honours_LeaveOpen(bool leaveOpen) + { + CancellationToken ct = TestContext.Current.CancellationToken; + byte[] bytes = CorruptZip(); + TrickleStream open = new(bytes); + Assert.Throws(() => Excel.Open(open, leaveOpen)); + Assert.Equal(leaveOpen, open.CanRead); + + TrickleStream openAsync = new(bytes); + await Assert.ThrowsAsync(async () => await Excel.OpenAsync(openAsync, leaveOpen, ct: ct)); + Assert.Equal(leaveOpen, openAsync.CanRead); + + TrickleStream xlsx = new(bytes); + Assert.Throws(() => Excel.FromXlsx(xlsx, leaveOpen)); + Assert.Equal(leaveOpen, xlsx.CanRead); + + TrickleStream xlsb = new(bytes); + Assert.Throws(() => Excel.FromXlsb(xlsb, leaveOpen)); + Assert.Equal(leaveOpen, xlsb.CanRead); + } + + [Fact] + public void A_Zip_Without_A_Workbook_Closes_The_Stream() + { + TrickleStream stream = new(ZipWithoutWorkbook()); + Assert.Throws(() => Excel.FromXlsx(stream, leaveOpen: false)); + Assert.False(stream.CanRead); + } + + [Fact] + public void A_Failed_Detection_Restores_The_Stream_Position() + { + TrickleStream stream = new(CorruptZip()); + Assert.Throws(() => Excel.DetectFileFormat(stream)); + Assert.Equal(0, stream.Position); + Assert.True(stream.CanRead); + } + + [Fact] + public async Task A_Failed_Asynchronous_Detection_Restores_The_Stream_Position() + { + TrickleStream stream = new(CorruptZip()); + await Assert.ThrowsAsync(async () => await Excel.DetectFileFormatAsync(stream, TestContext.Current.CancellationToken)); + Assert.Equal(0, stream.Position); + Assert.True(stream.CanRead); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Sources/ByteSourceStreamTests.cs b/tests/ExcelReader.Tests/Reader/Sources/ByteSourceStreamTests.cs new file mode 100644 index 00000000..25156abd --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Sources/ByteSourceStreamTests.cs @@ -0,0 +1,162 @@ +using ExcelReader.Core.Reader.Sources; + +namespace ExcelReader.Tests.Reader.Sources +{ + public sealed class ByteSourceStreamTests + { + private sealed class CountingSource(byte[] bytes) : ByteSource + { + internal int Reads { get; private set; } + + internal bool Disposed { get; private set; } + + internal override long Length => bytes.Length; + + internal override int Read(long offset, Span destination) + { + Reads++; + if (offset >= bytes.Length) + { + return 0; + } + int count = (int)Math.Min(destination.Length, bytes.Length - offset); + bytes.AsSpan((int)offset, count).CopyTo(destination); + return count; + } + + internal override ValueTask ReadAsync(long offset, Memory destination, CancellationToken ct) + { + return new ValueTask(Read(offset, destination.Span)); + } + + public override void Dispose() + { + Disposed = true; + } + } + + private static byte[] Payload(int length) + { + byte[] bytes = new byte[length]; + for (int i = 0; i < length; i++) + { + bytes[i] = (byte)((i * 31) + (i >> 8)); + } + return bytes; + } + + [Fact] + public void Reads_Exactly_Its_Range() + { + byte[] bytes = Payload(300_000); + using CountingSource source = new(bytes); + using ByteSourceStream stream = new(source, offset: 1000, length: 200_000); + using MemoryStream copy = new(); + stream.CopyTo(copy); + Assert.True(copy.ToArray().AsSpan().SequenceEqual(bytes.AsSpan(1000, 200_000))); + } + + [Fact] + public async Task ReadAsync_Reads_Exactly_Its_Range() + { + byte[] bytes = Payload(300_000); + using CountingSource source = new(bytes); + await using ByteSourceStream stream = new(source, offset: 1000, length: 200_000); + using MemoryStream copy = new(); + await stream.CopyToAsync(copy, TestContext.Current.CancellationToken); + Assert.True(copy.ToArray().AsSpan().SequenceEqual(bytes.AsSpan(1000, 200_000))); + } + + [Fact] + public void Small_Reads_Are_Served_From_One_Buffered_Fetch() + { + using CountingSource source = new(Payload(60_000)); + using ByteSourceStream stream = new(source, offset: 0, length: 60_000); + byte[] chunk = new byte[100]; + for (int i = 0; i < 600; i++) + { + stream.ReadExactly(chunk); + } + Assert.Equal(1, source.Reads); + } + + [Fact] + public void A_Large_Read_Bypasses_The_Buffer() + { + byte[] bytes = Payload(300_000); + using CountingSource source = new(bytes); + using ByteSourceStream stream = new(source, offset: 0, length: 300_000); + byte[] big = new byte[300_000]; + Assert.Equal(300_000, stream.Read(big)); + Assert.Equal(1, source.Reads); + Assert.True(big.AsSpan().SequenceEqual(bytes)); + } + + [Fact] + public void A_Source_Shorter_Than_The_Range_Ends_The_Stream() + { + using CountingSource source = new(Payload(100)); + using ByteSourceStream stream = new(source, offset: 50, length: 500); + byte[] buffer = new byte[1000]; + Assert.Equal(50, stream.Read(buffer)); + Assert.Equal(0, stream.Read(buffer)); + } + + [Fact] + public void Dispose_Does_Not_Dispose_The_Source() + { + using CountingSource source = new(Payload(10)); + new ByteSourceStream(source, 0, 10).Dispose(); + Assert.False(source.Disposed); + } + + [Fact] + public void Read_After_Dispose_Throws() + { + using CountingSource source = new(Payload(1000)); + ByteSourceStream stream = new(source, 0, 1000); + Assert.Equal(10, stream.Read(new byte[10])); + stream.Dispose(); + Assert.Throws(() => stream.Read(new byte[10])); + } + + [Fact] + public async Task ReadAsync_After_Dispose_Throws() + { + using CountingSource source = new(Payload(1000)); + ByteSourceStream stream = new(source, 0, 1000); + await stream.DisposeAsync(); + await Assert.ThrowsAsync( + async () => await stream.ReadAsync(new byte[10], TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task A_Large_Async_Read_Bypasses_The_Buffer() + { + byte[] bytes = Payload(300_000); + using CountingSource source = new(bytes); + await using ByteSourceStream stream = new(source, 0, 300_000); + byte[] big = new byte[300_000]; + Assert.Equal(300_000, await stream.ReadAsync(big, TestContext.Current.CancellationToken)); + Assert.Equal(1, source.Reads); + Assert.True(big.AsSpan().SequenceEqual(bytes)); + } + + [Fact] + public void An_Unbuffered_Stream_Reads_Straight_Into_The_Caller() + { + byte[] bytes = Payload(1000); + using CountingSource source = new(bytes); + using ByteSourceStream stream = new(source, 100, 800, buffered: false); + byte[] read = new byte[800]; + + Assert.Equal(10, stream.Read(read.AsSpan(0, 10))); + Assert.Equal(10, stream.Read(read.AsSpan(10, 10))); + Assert.Equal(2, source.Reads); + + stream.ReadExactly(read.AsSpan(20)); + Assert.True(read.AsSpan().SequenceEqual(bytes.AsSpan(100, 800))); + Assert.Equal(0, stream.Read(new byte[10])); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Sources/ByteSourceTests.cs b/tests/ExcelReader.Tests/Reader/Sources/ByteSourceTests.cs new file mode 100644 index 00000000..d9a0135c --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Sources/ByteSourceTests.cs @@ -0,0 +1,272 @@ +using ExcelReader.Core.Reader.Sources; + +namespace ExcelReader.Tests.Reader.Sources +{ + public sealed class ByteSourceTests : IDisposable + { + public enum Kind + { + Memory, + ExposedMemoryStream, + OpaqueMemoryStream, + Trickle, + FileStream, + FilePath, + NonSeekable, + } + + private static readonly byte[] Payload = BuildPayload(200_000); + + private readonly string _path = Path.Combine(Path.GetTempPath(), "excelreader-bytesource-" + Guid.NewGuid().ToString("N") + ".bin"); + + public ByteSourceTests() + { + File.WriteAllBytes(_path, Payload); + } + + public void Dispose() + { + File.Delete(_path); + } + + public static TheoryData Kinds => + [ + Kind.Memory, Kind.ExposedMemoryStream, Kind.OpaqueMemoryStream, Kind.Trickle, + Kind.FileStream, Kind.FilePath, Kind.NonSeekable, + ]; + + private static byte[] BuildPayload(int length) + { + byte[] bytes = new byte[length]; + uint state = 12345; + for (int i = 0; i < length; i++) + { + state = (state * 1664525) + 1013904223; + bytes[i] = (byte)(state >> 24); + } + return bytes; + } + + private ByteSource Open(Kind kind) + { + switch (kind) + { + case Kind.Memory: + return ByteSource.FromMemory(Payload); + case Kind.ExposedMemoryStream: + MemoryStream exposed = new(); + exposed.Write(Payload); + exposed.Position = 17; + return ByteSource.FromStream(exposed, leaveOpen: false); + case Kind.OpaqueMemoryStream: + return ByteSource.FromStream(new MemoryStream(Payload, writable: false), leaveOpen: false); + case Kind.Trickle: + return ByteSource.FromStream(new TrickleStream(Payload), leaveOpen: false); + case Kind.FileStream: + return ByteSource.FromStream(File.OpenRead(_path), leaveOpen: false); + case Kind.FilePath: + return ByteSource.OpenFile(_path, asynchronous: false); + default: + return ByteSource.FromStream(new NonSeekableStream(Payload), leaveOpen: false); + } + } + + [Theory] + [MemberData(nameof(Kinds))] + public void Length_Is_The_Whole_Payload(Kind kind) + { + using ByteSource source = Open(kind); + Assert.Equal(Payload.Length, source.Length); + } + + [Theory] + [MemberData(nameof(Kinds))] + public void ReadExactly_Returns_The_Bytes_At_The_Offset(Kind kind) + { + using ByteSource source = Open(kind); + byte[] read = new byte[5000]; + source.ReadExactly(150_000, read); + Assert.True(read.AsSpan().SequenceEqual(Payload.AsSpan(150_000, 5000))); + } + + [Theory] + [MemberData(nameof(Kinds))] + public async Task ReadExactlyAsync_Returns_The_Bytes_At_The_Offset(Kind kind) + { + using ByteSource source = Open(kind); + byte[] read = new byte[5000]; + await source.ReadExactlyAsync(150_000, read, TestContext.Current.CancellationToken); + Assert.True(read.AsSpan().SequenceEqual(Payload.AsSpan(150_000, 5000))); + } + + [Theory] + [MemberData(nameof(Kinds))] + public void Read_At_The_End_Returns_Zero(Kind kind) + { + using ByteSource source = Open(kind); + Assert.Equal(0, source.Read(Payload.Length, new byte[16])); + } + + [Theory] + [MemberData(nameof(Kinds))] + public void ReadExactly_Past_The_End_Throws(Kind kind) + { + using ByteSource source = Open(kind); + Assert.Throws(() => source.ReadExactly(Payload.Length - 10, new byte[11])); + } + + [Theory] + [MemberData(nameof(Kinds))] + public void TryGetMemory_Is_True_Only_When_The_Bytes_Are_In_Memory(Kind kind) + { + using ByteSource source = Open(kind); + bool expected = kind is Kind.Memory or Kind.ExposedMemoryStream or Kind.NonSeekable; + Assert.Equal(expected, source.TryGetMemory(out ReadOnlyMemory memory)); + if (expected) + { + Assert.True(memory.Span.SequenceEqual(Payload)); + } + } + + [Theory] + [MemberData(nameof(Kinds))] + public void Concurrent_Reads_Each_Get_Their_Own_Bytes(Kind kind) + { + using ByteSource source = Open(kind); + int iterations = kind == Kind.Trickle ? 20 : 200; + Parallel.For(0, 16, worker => + { + uint state = (uint)worker + 1; + byte[] buffer = new byte[4096]; + for (int i = 0; i < iterations; i++) + { + state = (state * 1664525) + 1013904223; + int offset = (int)(state % (uint)(Payload.Length - buffer.Length)); + source.ReadExactly(offset, buffer); + Assert.True(buffer.AsSpan().SequenceEqual(Payload.AsSpan(offset, buffer.Length))); + } + }); + } + + [Fact] + public void Dispose_Closes_A_Stream_It_Owns() + { + TrickleStream stream = new(Payload); + ByteSource.FromStream(stream, leaveOpen: false).Dispose(); + Assert.False(stream.CanRead); + } + + [Fact] + public void Dispose_Leaves_A_Borrowed_Stream_Open() + { + using TrickleStream stream = new(Payload); + ByteSource.FromStream(stream, leaveOpen: true).Dispose(); + Assert.True(stream.CanRead); + } + + [Fact] + public void A_Borrowed_FileStream_Stays_Usable() + { + using FileStream stream = File.OpenRead(_path); + ByteSource.FromStream(stream, leaveOpen: true).Dispose(); + Assert.Equal(Payload[0], stream.ReadByte()); + } + + [Fact] + public void Dispose_Releases_The_File_Handle() + { + ByteSource.OpenFile(_path, asynchronous: false).Dispose(); + File.Delete(_path); + Assert.False(File.Exists(_path)); + } + + [Fact] + public async Task FromStreamAsync_Buffers_A_NonSeekable_Stream() + { + using ByteSource source = await ByteSource.FromStreamAsync( + new NonSeekableStream(Payload), leaveOpen: false, TestContext.Current.CancellationToken); + Assert.True(source.TryGetMemory(out ReadOnlyMemory memory)); + Assert.True(memory.Span.SequenceEqual(Payload)); + } + + private sealed class InvertingMemoryStream(byte[] bytes) : MemoryStream(bytes, 0, bytes.Length, writable: false, publiclyVisible: true) + { + public override int Read(byte[] buffer, int offset, int count) + { + int read = base.Read(buffer, offset, count); + for (int i = 0; i < read; i++) + { + buffer[offset + i] ^= 0xFF; + } + return read; + } + + public override int Read(Span buffer) + { + byte[] scratch = new byte[buffer.Length]; + int read = Read(scratch, 0, scratch.Length); + scratch.AsSpan(0, read).CopyTo(buffer); + return read; + } + } + + [Fact] + public void A_MemoryStream_Subclass_Is_Read_Through_Its_Own_Read() + { + using ByteSource source = ByteSource.FromStream(new InvertingMemoryStream(Payload), leaveOpen: false); + Assert.False(source.TryGetMemory(out _)); + byte[] read = new byte[64]; + source.ReadExactly(100, read); + for (int i = 0; i < read.Length; i++) + { + Assert.Equal((byte)(Payload[100 + i] ^ 0xFF), read[i]); + } + } + + [Theory] + [MemberData(nameof(Kinds))] + public void A_Negative_Offset_Is_Rejected(Kind kind) + { + using ByteSource source = Open(kind); + Assert.Throws(() => source.Read(-1, new byte[4])); + } + + [Theory] + [MemberData(nameof(Kinds))] + public async Task ReadAsync_At_The_End_Returns_Zero(Kind kind) + { + using ByteSource source = Open(kind); + Assert.Equal(0, await source.ReadAsync(Payload.Length, new byte[16], TestContext.Current.CancellationToken)); + } + + [Fact] + public async Task A_Cancelled_Token_Stops_A_Stream_Read() + { + using ByteSource source = ByteSource.FromStream(new TrickleStream(Payload), leaveOpen: false); + using CancellationTokenSource cancelled = new(); + await cancelled.CancelAsync(); + await Assert.ThrowsAnyAsync( + async () => await source.ReadAsync(0, new byte[16], cancelled.Token)); + } + + [Fact] + public void An_Exposed_MemoryStream_With_A_Buffer_Offset_Reads_From_Its_Origin() + { + byte[] backing = new byte[Payload.Length + 100]; + Payload.CopyTo(backing, 100); + using ByteSource source = ByteSource.FromStream( + new MemoryStream(backing, 100, Payload.Length, writable: false, publiclyVisible: true), leaveOpen: false); + byte[] read = new byte[64]; + source.ReadExactly(10, read); + Assert.True(read.AsSpan().SequenceEqual(Payload.AsSpan(10, 64))); + } + + [Fact] + public async Task FromStreamAsync_Leaves_A_Borrowed_NonSeekable_Stream_Open() + { + using NonSeekableStream stream = new(Payload); + using ByteSource source = await ByteSource.FromStreamAsync(stream, leaveOpen: true, TestContext.Current.CancellationToken); + Assert.Equal(-1, stream.ReadByte()); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Xls/XlsLifetimeTests.cs b/tests/ExcelReader.Tests/Reader/Xls/XlsLifetimeTests.cs new file mode 100644 index 00000000..c4f31eaa --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Xls/XlsLifetimeTests.cs @@ -0,0 +1,88 @@ +using System.Globalization; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Xls; +using ExcelReader.Core.Writer.Xls; + +namespace ExcelReader.Tests.Reader.Xls +{ + public sealed class XlsLifetimeTests + { + private static byte[] BuildXls(int rows = 300) + { + using MemoryStream buffer = new(); + using (XlsWorkbookWriter workbook = XlsWorkbookWriter.Create(buffer, leaveOpen: true)) + { + XlsSheetWriter sheet = workbook.AddSheet("s"); + for (int r = 0; r < rows; r++) + { + using XlsRowWriter row = sheet.StartRow(); + row.Write("name-" + (r % 40).ToString(CultureInfo.InvariantCulture)); + row.Write(r); + } + sheet.End(); + sheet.Dispose(); + workbook.End(); + } + return buffer.ToArray(); + } + + private static List Drain(XlsReader.Enumerator e) + { + List values = []; + while (e.MoveNext()) + { + values.Add(e.Current[0].GetString()); + } + return values; + } + + [Fact] + public void GetEnumerator_After_Dispose_Throws() + { + XlsReader reader = Excel.FromXls(BuildXls()); + reader.Dispose(); + Assert.Throws(() => reader.GetEnumerator()); + Assert.Throws(() => reader.GetAsyncEnumerator(TestContext.Current.CancellationToken)); + } + + [Fact] + public void An_Enumerator_Outlives_Its_Reader() + { + byte[] bytes = BuildXls(); + List expected; + using (XlsReader reference = Excel.FromXls(bytes)) + { + using XlsReader.Enumerator all = reference.GetEnumerator(); + expected = Drain(all); + } + + TrickleStream stream = new(bytes); + XlsReader reader = Excel.FromXls(stream, leaveOpen: false); + XlsReader.Enumerator e = reader.GetEnumerator(); + Assert.True(e.MoveNext()); + + reader.Dispose(); + Assert.True(stream.CanRead); + + List rest = Drain(e); + Assert.Equal(expected.Skip(1), rest, StringComparer.Ordinal); + + e.Dispose(); + Assert.False(stream.CanRead); + } + + [Fact] + public void Disposing_Reader_And_Enumerator_Twice_Is_Harmless() + { + TrickleStream stream = new(BuildXls(rows: 20)); + XlsReader reader = Excel.FromXls(stream, leaveOpen: false); + XlsReader.Enumerator e = reader.GetEnumerator(); + e.Dispose(); + e.Dispose(); + Assert.True(stream.CanRead); + reader.Dispose(); + reader.Dispose(); + Assert.False(stream.CanRead); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Xlsb/XlsbEnumeratorTests.cs b/tests/ExcelReader.Tests/Reader/Xlsb/XlsbEnumeratorTests.cs index b37b39bb..e1b51ebf 100644 --- a/tests/ExcelReader.Tests/Reader/Xlsb/XlsbEnumeratorTests.cs +++ b/tests/ExcelReader.Tests/Reader/Xlsb/XlsbEnumeratorTests.cs @@ -42,10 +42,12 @@ public void EmptySheetYieldsNoRows() } [Fact] - public void EmptyRowIsSkipped() + public void EmptyRowIsYielded() { byte[] sheet = [.. B.Record(Brt.RowHdr), .. B.Record(Brt.EndSheetData)]; using var e = Open(BlankReader(), sheet); + Assert.True(e.MoveNext()); + Assert.Equal(0, e.Current.ColumnCount); Assert.False(e.MoveNext()); } @@ -60,6 +62,8 @@ .. B.Record(Brt.RowHdr), .. B.Record(Brt.CellBool, B.CellBool(0, 0, true)), ]; using var e = Open(BlankReader(), sheet); + Assert.True(e.MoveNext()); + Assert.Equal(0, e.Current.ColumnCount); Assert.False(e.MoveNext()); } @@ -236,7 +240,7 @@ .. B.Record(Brt.EndSheetData), } [Fact] - public void EmptyRowsBetweenDataRowsAreSkipped() + public void EmptyRowsBetweenDataRowsAreYielded() { byte[] sheet = [ @@ -251,6 +255,9 @@ .. B.Record(Brt.CellBool, B.CellBool(0, 0, false)), Assert.True(e.MoveNext()); Assert.Equal("1", e.Current[0].GetString()); + Assert.True(e.MoveNext()); + Assert.Equal(0, e.Current.ColumnCount); + Assert.True(e.MoveNext()); Assert.Equal("0", e.Current[0].GetString()); diff --git a/tests/ExcelReader.Tests/Reader/Xlsb/XlsbLifetimeTests.cs b/tests/ExcelReader.Tests/Reader/Xlsb/XlsbLifetimeTests.cs new file mode 100644 index 00000000..f409132a --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Xlsb/XlsbLifetimeTests.cs @@ -0,0 +1,88 @@ +using System.Globalization; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Xlsb; +using ExcelReader.Core.Writer.Xlsb; + +namespace ExcelReader.Tests.Reader.Xlsb +{ + public sealed class XlsbLifetimeTests + { + private static byte[] BuildXlsb(int rows = 300) + { + using MemoryStream buffer = new(); + using (XlsbWorkbookWriter workbook = XlsbWorkbookWriter.Create(buffer, leaveOpen: true, new XlsbWriterOptions { UseSharedStrings = true })) + { + XlsbSheetWriter sheet = workbook.AddSheet("s"); + for (int r = 0; r < rows; r++) + { + using XlsbRowWriter row = sheet.StartRow(); + row.Write("name-" + (r % 40).ToString(CultureInfo.InvariantCulture)); + row.Write(r); + } + sheet.End(); + sheet.Dispose(); + workbook.End(); + } + return buffer.ToArray(); + } + + private static List Drain(XlsbReader.Enumerator e) + { + List values = []; + while (e.MoveNext()) + { + values.Add(e.Current[0].GetString()); + } + return values; + } + + [Fact] + public void GetEnumerator_After_Dispose_Throws() + { + XlsbReader reader = Excel.FromXlsb(BuildXlsb()); + reader.Dispose(); + Assert.Throws(() => reader.GetEnumerator()); + Assert.Throws(() => reader.GetAsyncEnumerator(TestContext.Current.CancellationToken)); + } + + [Fact] + public void An_Enumerator_Outlives_Its_Reader() + { + byte[] bytes = BuildXlsb(); + List expected; + using (XlsbReader reference = Excel.FromXlsb(bytes)) + { + using XlsbReader.Enumerator all = reference.GetEnumerator(); + expected = Drain(all); + } + + TrickleStream stream = new(bytes); + XlsbReader reader = Excel.FromXlsb(stream, leaveOpen: false); + XlsbReader.Enumerator e = reader.GetEnumerator(); + Assert.True(e.MoveNext()); + + reader.Dispose(); + Assert.True(stream.CanRead); + + List rest = Drain(e); + Assert.Equal(expected.Skip(1), rest, StringComparer.Ordinal); + + e.Dispose(); + Assert.False(stream.CanRead); + } + + [Fact] + public void Disposing_Reader_And_Enumerator_Twice_Is_Harmless() + { + TrickleStream stream = new(BuildXlsb(rows: 20)); + XlsbReader reader = Excel.FromXlsb(stream, leaveOpen: false); + XlsbReader.Enumerator e = reader.GetEnumerator(); + e.Dispose(); + e.Dispose(); + Assert.True(stream.CanRead); + reader.Dispose(); + reader.Dispose(); + Assert.False(stream.CanRead); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Xlsx/XlsxLifetimeTests.cs b/tests/ExcelReader.Tests/Reader/Xlsx/XlsxLifetimeTests.cs new file mode 100644 index 00000000..d6e0ec4b --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Xlsx/XlsxLifetimeTests.cs @@ -0,0 +1,182 @@ +using System.Globalization; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Xlsx; +using ExcelReader.Core.Writer.Xlsx; + +namespace ExcelReader.Tests.Reader.Xlsx +{ + public sealed class XlsxLifetimeTests + { + private static byte[] BuildXlsx(int rows = 300) + { + using MemoryStream buffer = new(); + using (XlsxWorkbookWriter workbook = XlsxWorkbookWriter.Create(buffer, leaveOpen: true, new XlsxWriterOptions { UseSharedStrings = true })) + { + XlsxSheetWriter sheet = workbook.AddSheet("s"); + for (int r = 0; r < rows; r++) + { + using XlsxRowWriter row = sheet.StartRow(); + row.Write("name-" + (r % 40).ToString(CultureInfo.InvariantCulture)); + row.Write(r); + } + sheet.End(); + sheet.Dispose(); + workbook.End(); + } + return buffer.ToArray(); + } + + private static List Drain(XlsxReader.Enumerator e) + { + List values = []; + while (e.MoveNext()) + { + values.Add(e.Current[0].GetString()); + } + return values; + } + + [Fact] + public void A_Failed_Shared_String_Load_Fails_Every_Later_Enumeration() + { + using XlsxReader reader = Excel.FromXlsx(BuildXlsx(), new ExcelReaderOptions { MaxSharedStringBytes = 16 }); + Assert.Throws(() => reader.GetEnumerator()); + Assert.Throws(() => reader.GetEnumerator()); + } + + [Fact] + public async Task A_Failed_Shared_String_Load_Fails_Every_Later_Async_Enumeration() + { + using XlsxReader reader = Excel.FromXlsx( + new MemoryStream(BuildXlsx(), writable: false), leaveOpen: false, new ExcelReaderOptions { MaxSharedStringBytes = 16 }); + for (int attempt = 0; attempt < 2; attempt++) + { + await using XlsxReader.Enumerator e = reader.GetAsyncEnumerator(TestContext.Current.CancellationToken); + await Assert.ThrowsAsync(async () => await e.MoveNextAsync()); + } + } + + [Fact] + public void GetEnumerator_After_Dispose_Throws() + { + XlsxReader reader = Excel.FromXlsx(BuildXlsx()); + reader.Dispose(); + Assert.Throws(() => reader.GetEnumerator()); + Assert.Throws(() => reader.GetAsyncEnumerator(TestContext.Current.CancellationToken)); + } + + [Fact] + public void An_Enumerator_Outlives_Its_Reader() + { + byte[] bytes = BuildXlsx(); + List expected; + using (XlsxReader reference = Excel.FromXlsx(bytes)) + { + using XlsxReader.Enumerator all = reference.GetEnumerator(); + expected = Drain(all); + } + + TrickleStream stream = new(bytes); + XlsxReader reader = Excel.FromXlsx(stream, leaveOpen: false); + XlsxReader.Enumerator e = reader.GetEnumerator(); + Assert.True(e.MoveNext()); + + reader.Dispose(); + Assert.True(stream.CanRead); + + List rest = Drain(e); + Assert.Equal(expected.Skip(1), rest, StringComparer.Ordinal); + + e.Dispose(); + Assert.False(stream.CanRead); + } + + [Fact] + public void Disposing_Reader_And_Enumerator_Twice_Is_Harmless() + { + TrickleStream stream = new(BuildXlsx(rows: 20)); + XlsxReader reader = Excel.FromXlsx(stream, leaveOpen: false); + XlsxReader.Enumerator e = reader.GetEnumerator(); + e.Dispose(); + e.Dispose(); + Assert.True(stream.CanRead); + reader.Dispose(); + reader.Dispose(); + Assert.False(stream.CanRead); + } + + [Fact] + public void A_Failed_GetEnumerator_Does_Not_Keep_The_Reader_Alive() + { + TrickleStream stream = new(BuildXlsx(rows: 20)); + XlsxReader reader = Excel.FromXlsx(stream, leaveOpen: false, new ExcelReaderOptions { MaxSharedStringBytes = 16 }); + Assert.Throws(() => reader.GetEnumerator()); + reader.Dispose(); + Assert.False(stream.CanRead); + } + + [Fact] + public async Task A_Disposed_Deferred_Enumerator_Does_Not_Reopen_Its_Sheet() + { + TrickleStream stream = new(BuildXlsx(rows: 20)); + XlsxReader reader = Excel.FromXlsx(stream, leaveOpen: false); + XlsxReader.Enumerator e = reader.GetAsyncEnumerator(TestContext.Current.CancellationToken); + await e.DisposeAsync(); + Task move = Task.Run(async () => await e.MoveNextAsync(), TestContext.Current.CancellationToken); + Assert.Same(move, await Task.WhenAny(move, Task.Delay(TimeSpan.FromSeconds(3), TestContext.Current.CancellationToken))); + Assert.False(await move); + reader.Dispose(); + Assert.False(stream.CanRead); + } + + [Fact] + public async Task A_Cancelled_Shared_String_Load_Is_Undone_And_Loads_Again() + { + byte[] bytes = BuildXlsx(); + List expected; + using (XlsxReader reference = Excel.FromXlsx(bytes)) + { + using XlsxReader.Enumerator all = reference.GetEnumerator(); + expected = Drain(all); + } + + ReadOnlySpan entryName = "xl/sharedStrings.xml"u8; + int header = bytes.AsSpan().IndexOf(entryName) - 30; + long bodyStart = header + 30 + entryName.Length + BitConverter.ToUInt16(bytes, header + 28); + long bodyEnd = bodyStart + BitConverter.ToUInt32(bytes, header + 18); + + using CancellationTokenSource cts = new(); + int reads = 0; + long cancelledAt = -1; + TrickleStream stream = new(bytes); + XlsxReader reader = Excel.FromXlsx(stream, leaveOpen: false); + stream.OnRead = () => + { + if (++reads == 20) + { + cancelledAt = stream.Position; + cts.Cancel(); + } + }; + + await using (XlsxReader.Enumerator cancelled = reader.GetAsyncEnumerator(cts.Token)) + { + await Assert.ThrowsAnyAsync(async () => await cancelled.MoveNextAsync()); + } + Assert.InRange(cancelledAt, bodyStart, bodyEnd - 1); + + List actual = []; + await using (XlsxReader.Enumerator again = reader.GetAsyncEnumerator(TestContext.Current.CancellationToken)) + { + while (await again.MoveNextAsync()) + { + actual.Add(again.Current[0].GetString()); + } + } + Assert.Equal(expected, actual, StringComparer.Ordinal); + + reader.Dispose(); + Assert.False(stream.CanRead); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Zip/ZipIndexSourceTests.cs b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexSourceTests.cs new file mode 100644 index 00000000..c3f64a0c --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexSourceTests.cs @@ -0,0 +1,474 @@ +using System.IO.Compression; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Sources; +using ExcelReader.Core.Reader.Zip; + +namespace ExcelReader.Tests.Reader.Zip +{ + public sealed class ZipIndexSourceTests : IDisposable + { + public enum Kind + { + Memory, + OpaqueMemoryStream, + Trickle, + File, + } + + private static readonly string[] Names = ["deflated.txt", "stored.bin", "empty"]; + private static readonly int[] Sizes = [300_000, 70_000, 0]; + + private readonly List _paths = []; + + public void Dispose() + { + foreach (string path in _paths) + { + File.Delete(path); + } + } + + public static TheoryData KindsAndDescriptors + { + get + { + TheoryData data = []; + foreach (Kind kind in Enum.GetValues()) + { + data.Add(kind, false); + data.Add(kind, true); + } + return data; + } + } + + public static TheoryData Kinds => [Kind.Memory, Kind.OpaqueMemoryStream, Kind.Trickle, Kind.File]; + + private static byte[] Content(int entry, int scale = 1) + { + byte[] bytes = new byte[Sizes[entry] / scale]; + for (int i = 0; i < bytes.Length; i++) + { + bytes[i] = (byte)((i % 251) ^ (entry * 37)); + } + return bytes; + } + + private sealed class ForwardOnlyStream(Stream inner) : Stream + { + public override bool CanRead => false; + + public override bool CanSeek => false; + + public override bool CanWrite => true; + + public override long Length => throw new NotSupportedException(); + + public override long Position + { + get => throw new NotSupportedException(); + set => throw new NotSupportedException(); + } + + public override void Flush() + { + inner.Flush(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + throw new NotSupportedException(); + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotSupportedException(); + } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + inner.Write(buffer, offset, count); + } + } + + private static byte[] BuildZip(bool dataDescriptors, int scale = 1) + { + using MemoryStream buffer = new(); + using ForwardOnlyStream forward = new(buffer); + using (ZipArchive zip = new(dataDescriptors ? forward : buffer, ZipArchiveMode.Create, leaveOpen: true)) + { + for (int i = 0; i < Names.Length; i++) + { + ZipArchiveEntry entry = zip.CreateEntry(Names[i], i == 1 ? CompressionLevel.NoCompression : CompressionLevel.Optimal); + using Stream stream = entry.Open(); + stream.Write(Content(i, scale)); + } + } + return buffer.ToArray(); + } + + private ByteSource Open(Kind kind, byte[] zipBytes) + { + switch (kind) + { + case Kind.Memory: + return ByteSource.FromMemory(zipBytes); + case Kind.OpaqueMemoryStream: + return ByteSource.FromStream(new MemoryStream(zipBytes, writable: false), leaveOpen: false); + case Kind.Trickle: + return ByteSource.FromStream(new TrickleStream(zipBytes), leaveOpen: false); + default: + string path = Path.Combine(Path.GetTempPath(), "excelreader-zipindex-" + Guid.NewGuid().ToString("N") + ".zip"); + File.WriteAllBytes(path, zipBytes); + _paths.Add(path); + return ByteSource.OpenFile(path, asynchronous: false); + } + } + + private static DecompressedByteCounter Counter() + { + return new DecompressedByteCounter(ExcelReaderOptions.Default.MaxTotalDecompressedBytes); + } + + [Theory] + [MemberData(nameof(KindsAndDescriptors))] + public void Parts_Hold_The_Entry_Bytes(Kind kind, bool dataDescriptors) + { + using ZipIndex index = ZipIndex.Create(Open(kind, BuildZip(dataDescriptors)), ExcelReaderOptions.Default); + Assert.Equal(kind == Kind.Memory, index.HasMemory); + for (int i = 0; i < Names.Length; i++) + { + Assert.True(index.TryGetEntry(System.Text.Encoding.UTF8.GetBytes(Names[i]), out ZipEntryRef entry)); + using ZipPart part = index.OpenPart(entry, Counter()); + Assert.True(part.Memory.Span.SequenceEqual(Content(i))); + } + } + + [Theory] + [MemberData(nameof(KindsAndDescriptors))] + public void Entry_Streams_Yield_The_Entry_Bytes(Kind kind, bool dataDescriptors) + { + using ZipIndex index = ZipIndex.Create(Open(kind, BuildZip(dataDescriptors)), ExcelReaderOptions.Default); + for (int i = 0; i < Names.Length; i++) + { + Assert.True(index.TryGetEntry(System.Text.Encoding.UTF8.GetBytes(Names[i]), out ZipEntryRef entry)); + using Stream stream = index.OpenEntryStream(entry, Counter(), ExcelReaderOptions.Default); + using MemoryStream copy = new(); + stream.CopyTo(copy); + Assert.True(copy.ToArray().AsSpan().SequenceEqual(Content(i))); + } + } + + [Theory] + [MemberData(nameof(KindsAndDescriptors))] + public async Task Async_Open_Matches_Sync_Open(Kind kind, bool dataDescriptors) + { + CancellationToken ct = TestContext.Current.CancellationToken; + using ZipIndex index = await ZipIndex.CreateAsync(Open(kind, BuildZip(dataDescriptors)), ExcelReaderOptions.Default, ct); + for (int i = 0; i < Names.Length; i++) + { + byte[] name = System.Text.Encoding.UTF8.GetBytes(Names[i]); + using ZipPart part = await index.OpenPartOrDefaultAsync(name, Counter(), ct); + Assert.True(part.Memory.Span.SequenceEqual(Content(i))); + + Assert.True(index.TryGetEntry(name, out ZipEntryRef entry)); + await using Stream stream = await index.OpenEntryStreamAsync(entry, Counter(), ExcelReaderOptions.Default, ct); + using MemoryStream copy = new(); + await stream.CopyToAsync(copy, ct); + Assert.True(copy.ToArray().AsSpan().SequenceEqual(Content(i))); + } + } + + [Theory] + [MemberData(nameof(Kinds))] + public void Entries_Open_Concurrently(Kind kind) + { + using ZipIndex index = ZipIndex.Create(Open(kind, BuildZip(dataDescriptors: false, scale: kind == Kind.Trickle ? 20 : 1)), ExcelReaderOptions.Default); + int scale = kind == Kind.Trickle ? 20 : 1; + Parallel.For(0, 24, worker => + { + int which = worker % 2; + Assert.True(index.TryGetEntry(System.Text.Encoding.UTF8.GetBytes(Names[which]), out ZipEntryRef entry)); + using Stream stream = index.OpenEntryStream(entry, Counter(), ExcelReaderOptions.Default); + using MemoryStream copy = new(); + stream.CopyTo(copy); + Assert.True(copy.ToArray().AsSpan().SequenceEqual(Content(which, scale))); + }); + } + + [Theory] + [MemberData(nameof(Kinds))] + public void A_File_Cut_Short_Is_Invalid_Data(Kind kind) + { + byte[] zipBytes = BuildZip(dataDescriptors: false, scale: 20); + Assert.Throws( + () => ZipIndex.Create(Open(kind, zipBytes[..(zipBytes.Length / 2)]), ExcelReaderOptions.Default)); + Assert.Throws( + () => ZipIndex.Create(Open(kind, []), ExcelReaderOptions.Default)); + } + + [Theory] + [MemberData(nameof(Kinds))] + public void A_Stored_Entry_Whose_Sizes_Disagree_Is_Invalid_Data(Kind kind) + { + byte[] zipBytes = BuildZip(dataDescriptors: false, scale: 20); + byte[] name = System.Text.Encoding.UTF8.GetBytes(Names[1]); + int record = -1; + for (int i = 0; i + 46 + name.Length <= zipBytes.Length; i++) + { + if (zipBytes.AsSpan(i, 4).SequenceEqual([0x50, 0x4B, 0x01, 0x02]) && zipBytes.AsSpan(i + 46, name.Length).SequenceEqual(name)) + { + record = i; + break; + } + } + Assert.True(record >= 0, "central directory record not found"); + zipBytes.AsSpan(record + 24, 4).Clear(); + + using ZipIndex index = ZipIndex.Create(Open(kind, zipBytes), ExcelReaderOptions.Default); + Assert.True(index.TryGetEntry(name, out ZipEntryRef entry)); + Assert.Throws(() => index.OpenPart(entry, Counter()).Dispose()); + Assert.Throws(() => index.OpenEntryStream(entry, Counter(), ExcelReaderOptions.Default).Dispose()); + } + + [Theory] + [MemberData(nameof(Kinds))] + public void Bytes_After_The_End_Of_Central_Directory_Are_Ignored(Kind kind) + { + byte[] zipBytes = BuildZip(dataDescriptors: false); + foreach (int extra in (int[])[1, 100]) + { + byte[] padded = new byte[zipBytes.Length + extra]; + zipBytes.CopyTo(padded, 0); + padded.AsSpan(zipBytes.Length).Fill(0xAB); + + using ZipIndex index = ZipIndex.Create(Open(kind, padded), ExcelReaderOptions.Default); + for (int i = 0; i < Names.Length; i++) + { + Assert.True(index.TryGetEntry(System.Text.Encoding.UTF8.GetBytes(Names[i]), out ZipEntryRef entry)); + using ZipPart part = index.OpenPart(entry, Counter()); + Assert.True(part.Memory.Span.SequenceEqual(Content(i))); + } + } + } + + [Fact] + public void A_Failed_Create_Disposes_The_Source() + { + TrickleStream stream = new(new byte[100]); + Assert.Throws( + () => ZipIndex.Create(ByteSource.FromStream(stream, leaveOpen: false), ExcelReaderOptions.Default)); + Assert.False(stream.CanRead); + } + + private enum Access + { + Part, + Stream, + AsyncPart, + AsyncStream, + } + + private static async Task ReadEntryAsync(ZipIndex index, ZipEntryRef entry, Access access, CancellationToken ct) + { + switch (access) + { + case Access.Part: + using (ZipPart part = index.OpenPart(entry, Counter())) + { + return part.Memory.ToArray(); + } + case Access.AsyncPart: + using (ZipPart part = await index.OpenPartAsync(entry, Counter(), ct)) + { + return part.Memory.ToArray(); + } + case Access.Stream: + using (Stream stream = index.OpenEntryStream(entry, Counter(), ExcelReaderOptions.Default)) + { + using MemoryStream copy = new(); + stream.CopyTo(copy); + return copy.ToArray(); + } + default: + await using (Stream stream = await index.OpenEntryStreamAsync(entry, Counter(), ExcelReaderOptions.Default, ct)) + { + using MemoryStream copy = new(); + await stream.CopyToAsync(copy, ct); + return copy.ToArray(); + } + } + } + + private static async Task OutcomeAsync(Func open, Access access, CancellationToken ct) + { + try + { + using ZipIndex index = access is Access.AsyncPart or Access.AsyncStream + ? await ZipIndex.CreateAsync(open(), ExcelReaderOptions.Default, ct) + : ZipIndex.Create(open(), ExcelReaderOptions.Default); + System.Text.StringBuilder text = new(); + foreach (string name in Names) + { + if (!index.TryGetEntry(System.Text.Encoding.UTF8.GetBytes(name), out ZipEntryRef entry)) + { + text.Append("missing;"); + continue; + } + try + { + byte[] bytes = await ReadEntryAsync(index, entry, access, ct); + text.Append(Convert.ToHexString(System.Security.Cryptography.SHA256.HashData(bytes))).Append(';'); + } + catch (Exception ex) when (ex is InvalidDataException or NotSupportedException or ExcelLimitExceededException) + { + text.Append(ex.GetType().Name).Append(';'); + } + } + return text.ToString(); + } + catch (Exception ex) when (ex is InvalidDataException or NotSupportedException or ExcelLimitExceededException) + { + return ex.GetType().Name; + } + } + + [Fact] + public async Task Mutated_Bytes_Give_The_Same_Outcome_From_Memory_And_From_A_Stream() + { + CancellationToken ct = TestContext.Current.CancellationToken; + byte[] pristine = BuildZip(dataDescriptors: false, scale: 1000); + uint state = 2024; + for (int i = 0; i < 3000; i++) + { + byte[] mutated = (byte[])pristine.Clone(); + state = (state * 1664525) + 1013904223; + int position = (int)(state % (uint)mutated.Length); + state = (state * 1664525) + 1013904223; + mutated[position] = (byte)(state >> 24); + + foreach (Access access in Enum.GetValues()) + { + Access reference = access is Access.Part or Access.AsyncPart ? Access.Part : Access.Stream; + string fromMemory = await OutcomeAsync(() => ByteSource.FromMemory(mutated), reference, ct); + string fromStream = await OutcomeAsync( + () => ByteSource.FromStream(new MemoryStream(mutated, writable: false), leaveOpen: false), access, ct); + Assert.True(string.Equals(fromMemory, fromStream, StringComparison.Ordinal), $"mutation {i} at byte {position}, {access}: memory={fromMemory} stream={fromStream}"); + } + } + } + + private static int CentralRecordOffset(byte[] zipBytes, string entryName) + { + byte[] name = System.Text.Encoding.UTF8.GetBytes(entryName); + for (int i = 0; i + 46 + name.Length <= zipBytes.Length; i++) + { + if (zipBytes.AsSpan(i, 4).SequenceEqual([0x50, 0x4B, 0x01, 0x02]) && zipBytes.AsSpan(i + 46, name.Length).SequenceEqual(name)) + { + return i; + } + } + throw new InvalidOperationException("central directory record not found"); + } + + private static byte[] ZipDeclaringTooMuchData() + { + byte[] zipBytes = BuildZip(dataDescriptors: false, scale: 20); + int record = CentralRecordOffset(zipBytes, Names[0]); + System.Buffers.Binary.BinaryPrimitives.WriteUInt32LittleEndian(zipBytes.AsSpan(record + 24), (uint)(Sizes[0] / 20) + 64); + return zipBytes; + } + + [Theory] + [MemberData(nameof(Kinds))] + public void A_Deflated_Entry_Shorter_Than_Declared_Is_Invalid_Data(Kind kind) + { + using ZipIndex index = ZipIndex.Create(Open(kind, ZipDeclaringTooMuchData()), ExcelReaderOptions.Default); + Assert.True(index.TryGetEntry(System.Text.Encoding.UTF8.GetBytes(Names[0]), out ZipEntryRef entry)); + InvalidDataException ex = Assert.Throws(() => index.OpenPart(entry, Counter()).Dispose()); + Assert.Contains("less data", ex.Message, StringComparison.Ordinal); + } + + [Theory] + [MemberData(nameof(Kinds))] + public async Task A_Deflated_Entry_Shorter_Than_Declared_Is_Invalid_Data_Asynchronously(Kind kind) + { + CancellationToken ct = TestContext.Current.CancellationToken; + using ZipIndex index = await ZipIndex.CreateAsync(Open(kind, ZipDeclaringTooMuchData()), ExcelReaderOptions.Default, ct); + Assert.True(index.TryGetEntry(System.Text.Encoding.UTF8.GetBytes(Names[0]), out ZipEntryRef entry)); + InvalidDataException ex = await Assert.ThrowsAsync( + async () => (await index.OpenPartAsync(entry, Counter(), ct)).Dispose()); + Assert.Contains("less data", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void Test_Workbooks_Match_ZipArchive_Entry_For_Entry() + { + string dataDirectory = Path.Combine(AppContext.BaseDirectory, "data"); + int checkedFiles = 0; + foreach (string path in Directory.EnumerateFiles(dataDirectory, "*.xls?", SearchOption.AllDirectories)) + { + byte[] bytes = File.ReadAllBytes(path); + if (bytes.Length < 4 || bytes[0] != (byte)'P' || bytes[1] != (byte)'K') + { + continue; + } + using ZipArchive expected = new(new MemoryStream(bytes, writable: false), ZipArchiveMode.Read); + using ZipIndex index = ZipIndex.Create(ByteSource.OpenFile(path, asynchronous: false), ExcelReaderOptions.Default); + foreach (ZipArchiveEntry expectedEntry in expected.Entries) + { + if (expectedEntry.FullName.EndsWith('/')) + { + continue; + } + Assert.True(index.TryGetEntry(System.Text.Encoding.UTF8.GetBytes(expectedEntry.FullName), out ZipEntryRef entry), expectedEntry.FullName); + using Stream expectedStream = expectedEntry.Open(); + using MemoryStream expectedBytes = new(); + expectedStream.CopyTo(expectedBytes); + using ZipPart part = index.OpenPart(entry, Counter()); + Assert.True(part.Memory.Span.SequenceEqual(expectedBytes.ToArray()), path + "!" + expectedEntry.FullName); + } + checkedFiles++; + } + Assert.True(checkedFiles > 0, "no ZIP-based workbook found under " + dataDirectory); + } + + [Theory] + [MemberData(nameof(Kinds))] + public void A_Disposed_Index_Refuses_Every_Open(Kind kind) + { + ZipIndex index = ZipIndex.Create(Open(kind, BuildZip(dataDescriptors: false, scale: 20)), ExcelReaderOptions.Default); + Assert.True(index.TryGetEntry(System.Text.Encoding.UTF8.GetBytes(Names[0]), out ZipEntryRef entry)); + index.Dispose(); + + Assert.Throws(() => index.TryGetEntry(System.Text.Encoding.UTF8.GetBytes(Names[0]), out _)); + Assert.Throws(() => index.OpenPart(entry, Counter())); + Assert.Throws(() => index.OpenEntryStream(entry, Counter(), ExcelReaderOptions.Default)); + } + + [Fact] + public async Task A_Failed_CreateAsync_Disposes_The_Source() + { + TrickleStream stream = new(new byte[100]); + await Assert.ThrowsAsync(async () => await ZipIndex.CreateAsync( + ByteSource.FromStream(stream, leaveOpen: false), ExcelReaderOptions.Default, TestContext.Current.CancellationToken)); + Assert.False(stream.CanRead); + } + + [Fact] + public async Task A_Cancelled_CreateAsync_Disposes_The_Source() + { + TrickleStream stream = new(BuildZip(dataDescriptors: false, scale: 20)); + using CancellationTokenSource cancelled = new(); + await cancelled.CancelAsync(); + await Assert.ThrowsAnyAsync(async () => await ZipIndex.CreateAsync( + ByteSource.FromStream(stream, leaveOpen: false), ExcelReaderOptions.Default, cancelled.Token)); + Assert.False(stream.CanRead); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Xlsx/ZipMemoryIndexTests.cs b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs similarity index 65% rename from tests/ExcelReader.Tests/Reader/Xlsx/ZipMemoryIndexTests.cs rename to tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs index 584c76df..8a765067 100644 --- a/tests/ExcelReader.Tests/Reader/Xlsx/ZipMemoryIndexTests.cs +++ b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs @@ -4,11 +4,12 @@ using System.Runtime.InteropServices; using System.Text; using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Sources; using ExcelReader.Core.Reader.Zip; -namespace ExcelReader.Tests.Reader.Xlsx +namespace ExcelReader.Tests.Reader.Zip { - public class ZipMemoryIndexTests + public class ZipIndexTests { [Fact] public void OpenPartOnDeflatedEntryMatchesStreamedRead() @@ -17,7 +18,7 @@ public void OpenPartOnDeflatedEntryMatchesStreamedRead() byte[] zipBytes = built.ToArray(); byte[] expected = ReadViaZipArchive(zipBytes, "xl/workbook.xml"); - using ZipMemoryIndex index = ZipMemoryIndex.Create(zipBytes, ExcelReaderOptions.Default); + using ZipIndex index = ZipIndex.Create(zipBytes, ExcelReaderOptions.Default); Assert.True(index.TryGetEntry("xl/workbook.xml"u8, out ZipEntryRef entry)); Assert.Equal((ushort)8, entry.Method); @@ -31,7 +32,7 @@ public void OpenPartOnStoredEntryAliasesTheSourceBufferWithoutCopying() byte[] payload = Encoding.UTF8.GetBytes("hello world, stored and not deflated"); byte[] zipBytes = BuildZipWithOneEntry("hello.txt", payload, CompressionLevel.NoCompression); - using ZipMemoryIndex index = ZipMemoryIndex.Create(zipBytes, ExcelReaderOptions.Default); + using ZipIndex index = ZipIndex.Create(zipBytes, ExcelReaderOptions.Default); Assert.True(index.TryGetEntry("hello.txt"u8, out ZipEntryRef entry)); Assert.Equal((ushort)0, entry.Method); @@ -47,7 +48,7 @@ public void TryGetEntryReturnsFalseForAnAbsentName() using MemoryStream built = WorkbookBuilder.Build("""1"""); byte[] zipBytes = built.ToArray(); - using ZipMemoryIndex index = ZipMemoryIndex.Create(zipBytes, ExcelReaderOptions.Default); + using ZipIndex index = ZipIndex.Create(zipBytes, ExcelReaderOptions.Default); Assert.False(index.TryGetEntry("xl/doesNotExist.xml"u8, out _)); } @@ -69,7 +70,7 @@ public void CreateThrowsOnDuplicateCentralDirectoryEntryNames() byte[] zipBytes = ms.ToArray(); InvalidDataException ex = Assert.Throws( - () => ZipMemoryIndex.Create(zipBytes, ExcelReaderOptions.Default)); + () => ZipIndex.Create(zipBytes, ExcelReaderOptions.Default)); Assert.Contains("duplicate", ex.Message, StringComparison.OrdinalIgnoreCase); } @@ -80,7 +81,7 @@ public void OpenPartThrowsWhenLocalHeaderNameDoesNotMatchCentralDirectory() byte[] zipBytes = BuildZipWithOneEntry("hello.txt", payload, CompressionLevel.NoCompression); long localHeaderOffset; - using (ZipMemoryIndex index = ZipMemoryIndex.Create(zipBytes, ExcelReaderOptions.Default)) + using (ZipIndex index = ZipIndex.Create(zipBytes, ExcelReaderOptions.Default)) { Assert.True(index.TryGetEntry("hello.txt"u8, out ZipEntryRef entry)); localHeaderOffset = entry.LocalHeaderOffset; @@ -90,7 +91,7 @@ public void OpenPartThrowsWhenLocalHeaderNameDoesNotMatchCentralDirectory() Assert.Equal((byte)'h', zipBytes[nameOffset]); zipBytes[nameOffset] = (byte)'j'; - using ZipMemoryIndex mutatedIndex = ZipMemoryIndex.Create(zipBytes, ExcelReaderOptions.Default); + using ZipIndex mutatedIndex = ZipIndex.Create(zipBytes, ExcelReaderOptions.Default); Assert.True(mutatedIndex.TryGetEntry("hello.txt"u8, out ZipEntryRef mutatedEntry)); InvalidDataException ex = Assert.Throws( () => mutatedIndex.OpenPart(mutatedEntry, new DecompressedByteCounter(0))); @@ -104,7 +105,7 @@ public void OpenPartThrowsWhenDeclaredSizeExceedsTheRemainingBudget() byte[] zipBytes = built.ToArray(); PatchCentralDirectoryUInt32(zipBytes, "xl/workbook.xml", fieldOffset: 24, 50_000_000); - using ZipMemoryIndex index = ZipMemoryIndex.Create(zipBytes, ExcelReaderOptions.Default); + using ZipIndex index = ZipIndex.Create(zipBytes, ExcelReaderOptions.Default); Assert.True(index.TryGetEntry("xl/workbook.xml"u8, out ZipEntryRef entry)); var counter = new DecompressedByteCounter(4096, nameof(ExcelReaderOptions.MaxTotalDecompressedBytes)); @@ -131,7 +132,7 @@ public void CreateThrowsWhenEntryCountExceedsMaxZipEntries() var options = new ExcelReaderOptions { MaxZipEntries = 10 }; ExcelLimitExceededException ex = Assert.Throws( - () => ZipMemoryIndex.Create(zipBytes, options)); + () => ZipIndex.Create(zipBytes, options)); Assert.Equal(nameof(ExcelReaderOptions.MaxZipEntries), ex.LimitName); } @@ -142,7 +143,7 @@ public void CreateThrowsForAnEncryptedEntry() byte[] zipBytes = built.ToArray(); PatchCentralDirectoryUInt16(zipBytes, "xl/workbook.xml", fieldOffset: 8, 0x0001); - Assert.Throws(() => ZipMemoryIndex.Create(zipBytes, ExcelReaderOptions.Default)); + Assert.Throws(() => ZipIndex.Create(zipBytes, ExcelReaderOptions.Default)); } [Fact] @@ -152,7 +153,7 @@ public void OpenPartThrowsForAnUnsupportedCompressionMethod() byte[] zipBytes = built.ToArray(); PatchCentralDirectoryUInt16(zipBytes, "xl/workbook.xml", fieldOffset: 10, 12); - using ZipMemoryIndex index = ZipMemoryIndex.Create(zipBytes, ExcelReaderOptions.Default); + using ZipIndex index = ZipIndex.Create(zipBytes, ExcelReaderOptions.Default); Assert.True(index.TryGetEntry("xl/workbook.xml"u8, out ZipEntryRef entry)); Assert.Throws(() => index.OpenPart(entry, new DecompressedByteCounter(0))); } @@ -161,11 +162,11 @@ public void OpenPartThrowsForAnUnsupportedCompressionMethod() public void CreateThrowsWhenNoEndOfCentralDirectoryRecordExists() { byte[] notAZip = Encoding.UTF8.GetBytes("this is not a zip file at all"); - Assert.Throws(() => ZipMemoryIndex.Create(notAZip, ExcelReaderOptions.Default)); + Assert.Throws(() => ZipIndex.Create(notAZip, ExcelReaderOptions.Default)); } [Fact] - public void MutatedZipBytesNeverCrashTheMemoryIndex() + public void MutatedZipBytesNeverCrashTheIndex() { using MemoryStream built = WorkbookBuilder.BuildMultiSheet( sheets: @@ -205,7 +206,7 @@ public void MutatedZipBytesNeverCrashTheMemoryIndex() private static void OpenAllPartsAndDrain(byte[] bytes) { - using ZipMemoryIndex index = ZipMemoryIndex.Create(bytes, ExcelReaderOptions.Default); + using ZipIndex index = ZipIndex.Create(bytes, ExcelReaderOptions.Default); var counter = new DecompressedByteCounter(ExcelReaderOptions.Default.MaxTotalDecompressedBytes); foreach (string name in (string[])["xl/workbook.xml", "xl/sharedStrings.xml", "xl/styles.xml", "xl/worksheets/sheet1.xml"]) { @@ -243,7 +244,7 @@ public void HugeZip64EocdLocatorOffsetThrowsInsteadOfWrapping() WriteEocd(bytes, 20, declaredCount: 0xFFFF, cdSize: Zip64SentinelU32, cdOffset: Zip64SentinelU32); InvalidDataException ex = Assert.Throws( - () => ZipMemoryIndex.Create(bytes, ExcelReaderOptions.Default)); + () => ZipIndex.Create(bytes, ExcelReaderOptions.Default)); Assert.Contains("ZIP64", ex.Message, StringComparison.Ordinal); } @@ -259,7 +260,7 @@ public void HugeZip64CentralDirectorySizeThrowsInsteadOfWrapping() WriteEocd(bytes, 76, declaredCount: 0xFFFF, cdSize: Zip64SentinelU32, cdOffset: Zip64SentinelU32); InvalidDataException ex = Assert.Throws( - () => ZipMemoryIndex.Create(bytes, ExcelReaderOptions.Default)); + () => ZipIndex.Create(bytes, ExcelReaderOptions.Default)); Assert.Contains("central directory", ex.Message, StringComparison.OrdinalIgnoreCase); } @@ -297,7 +298,7 @@ public void NegativeZip64UncompressedSizeThrowsInvalidDataException() byte[] bytes = BuildCdWithZip64Field(compressedSize: 5, uncompressedSize: Zip64SentinelU32, zip64Value: -1); InvalidDataException ex = Assert.Throws( - () => ZipMemoryIndex.Create(bytes, ExcelReaderOptions.Default)); + () => ZipIndex.Create(bytes, ExcelReaderOptions.Default)); Assert.Contains("negative", ex.Message, StringComparison.OrdinalIgnoreCase); } @@ -307,7 +308,7 @@ public void NegativeZip64CompressedSizeThrowsInvalidDataException() byte[] bytes = BuildCdWithZip64Field(compressedSize: Zip64SentinelU32, uncompressedSize: 5, zip64Value: -1); InvalidDataException ex = Assert.Throws( - () => ZipMemoryIndex.Create(bytes, ExcelReaderOptions.Default)); + () => ZipIndex.Create(bytes, ExcelReaderOptions.Default)); Assert.Contains("negative", ex.Message, StringComparison.OrdinalIgnoreCase); } @@ -318,7 +319,7 @@ public void ValidZip64ArchiveOpensTheStoredEntry() { Zip64Layout zip = BuildStoredZip64(Zip64Payload); - using ZipMemoryIndex index = ZipMemoryIndex.Create(zip.Bytes, ExcelReaderOptions.Default); + using ZipIndex index = ZipIndex.Create(zip.Bytes, ExcelReaderOptions.Default); Assert.True(index.TryGetEntry("a.txt"u8, out ZipEntryRef entry)); Assert.Equal(0, entry.LocalHeaderOffset); Assert.Equal(Zip64Payload.Length, entry.CompressedSize); @@ -344,7 +345,7 @@ public void HugeZip64LocalHeaderOffsetThrowsInvalidDataOnOpen(long localOffset) { Zip64Layout zip = BuildStoredZip64(Zip64Payload, localOffset: localOffset); - using ZipMemoryIndex index = ZipMemoryIndex.Create(zip.Bytes, ExcelReaderOptions.Default); + using ZipIndex index = ZipIndex.Create(zip.Bytes, ExcelReaderOptions.Default); Assert.True(index.TryGetEntry("a.txt"u8, out ZipEntryRef entry)); Assert.Equal(localOffset, entry.LocalHeaderOffset); InvalidDataException ex = Assert.Throws( @@ -358,7 +359,7 @@ public void LocalHeaderOffsetJustInsideTheFileEndThrowsInvalidDataOnOpen() Zip64Layout probe = BuildStoredZip64(Zip64Payload); Zip64Layout zip = BuildStoredZip64(Zip64Payload, localOffset: probe.Bytes.Length - 29); - using ZipMemoryIndex index = ZipMemoryIndex.Create(zip.Bytes, ExcelReaderOptions.Default); + using ZipIndex index = ZipIndex.Create(zip.Bytes, ExcelReaderOptions.Default); Assert.True(index.TryGetEntry("a.txt"u8, out ZipEntryRef entry)); InvalidDataException ex = Assert.Throws( () => index.OpenPart(entry, new DecompressedByteCounter(0))); @@ -374,7 +375,7 @@ public void HugeZip64CompressedSizeThrowsInvalidDataOnOpen(long compressedSize) { Zip64Layout zip = BuildStoredZip64(Zip64Payload, compressedSize: compressedSize); - using ZipMemoryIndex index = ZipMemoryIndex.Create(zip.Bytes, ExcelReaderOptions.Default); + using ZipIndex index = ZipIndex.Create(zip.Bytes, ExcelReaderOptions.Default); Assert.True(index.TryGetEntry("a.txt"u8, out ZipEntryRef entry)); Assert.Equal(compressedSize, entry.CompressedSize); InvalidDataException ex = Assert.Throws( @@ -390,8 +391,8 @@ public void CompressedSizeEndingExactlyAtTheFileEndIsAcceptedAndOneMoreIsRejecte Zip64Layout probe = BuildStoredZip64(Zip64Payload); long exact = probe.Bytes.Length - probe.DataOffset; - Zip64Layout fits = BuildStoredZip64(Zip64Payload, compressedSize: exact); - using (ZipMemoryIndex index = ZipMemoryIndex.Create(fits.Bytes, ExcelReaderOptions.Default)) + Zip64Layout fits = BuildStoredZip64(Zip64Payload, compressedSize: exact, uncompressedSize: exact); + using (ZipIndex index = ZipIndex.Create(fits.Bytes, ExcelReaderOptions.Default)) { Assert.True(index.TryGetEntry("a.txt"u8, out ZipEntryRef entry)); using ZipPart part = index.OpenPart(entry, new DecompressedByteCounter(0)); @@ -399,7 +400,7 @@ public void CompressedSizeEndingExactlyAtTheFileEndIsAcceptedAndOneMoreIsRejecte } Zip64Layout overruns = BuildStoredZip64(Zip64Payload, compressedSize: exact + 1); - using (ZipMemoryIndex index = ZipMemoryIndex.Create(overruns.Bytes, ExcelReaderOptions.Default)) + using (ZipIndex index = ZipIndex.Create(overruns.Bytes, ExcelReaderOptions.Default)) { Assert.True(index.TryGetEntry("a.txt"u8, out ZipEntryRef entry)); Assert.Throws(() => index.OpenPart(entry, new DecompressedByteCounter(0))); @@ -414,7 +415,7 @@ public void Zip64LocalHeaderOffsetAboveLongMaxIsRejectedAsNegative(ulong localOf Zip64Layout zip = BuildStoredZip64(Zip64Payload, localOffset: unchecked((long)localOffset)); InvalidDataException ex = Assert.Throws( - () => ZipMemoryIndex.Create(zip.Bytes, ExcelReaderOptions.Default)); + () => ZipIndex.Create(zip.Bytes, ExcelReaderOptions.Default)); Assert.Contains("negative", ex.Message, StringComparison.OrdinalIgnoreCase); } @@ -427,7 +428,7 @@ public void Zip64LocatorOffsetAboveLongMaxIsRejectedNotIgnored(ulong zip64EocdOf BinaryPrimitives.WriteUInt64LittleEndian(zip.Bytes.AsSpan(zip.LocatorOffset + 8), zip64EocdOffset); InvalidDataException ex = Assert.Throws( - () => ZipMemoryIndex.Create(zip.Bytes, ExcelReaderOptions.Default)); + () => ZipIndex.Create(zip.Bytes, ExcelReaderOptions.Default)); Assert.Contains("ZIP64", ex.Message, StringComparison.Ordinal); } @@ -446,7 +447,7 @@ public void Zip64CentralDirectoryBoundsThatWouldOverflowAreRejected(long cdOffse BinaryPrimitives.WriteInt64LittleEndian(zip.Bytes.AsSpan(zip.Zip64EocdOffset + 48), cdOffset); InvalidDataException ex = Assert.Throws( - () => ZipMemoryIndex.Create(zip.Bytes, ExcelReaderOptions.Default)); + () => ZipIndex.Create(zip.Bytes, ExcelReaderOptions.Default)); Assert.Contains("central directory", ex.Message, StringComparison.OrdinalIgnoreCase); } @@ -459,7 +460,7 @@ public void Zip64CentralDirectorySizeOneByteLongerThanItsRecordsIsRejected() Assert.Equal(zip.Zip64EocdOffset, cdOffset + cdSize); BinaryPrimitives.WriteInt64LittleEndian(zip.Bytes.AsSpan(zip.Zip64EocdOffset + 40), cdSize + 1); - Assert.Throws(() => ZipMemoryIndex.Create(zip.Bytes, ExcelReaderOptions.Default)); + Assert.Throws(() => ZipIndex.Create(zip.Bytes, ExcelReaderOptions.Default)); } [Theory] @@ -473,7 +474,7 @@ public void CentralDirectoryVariableFieldsRunningPastTheDirectoryEndAreRejected( PatchCentralDirectoryUInt16(zipBytes, "hello.txt", fieldOffset, value); InvalidDataException ex = Assert.Throws( - () => ZipMemoryIndex.Create(zipBytes, ExcelReaderOptions.Default)); + () => ZipIndex.Create(zipBytes, ExcelReaderOptions.Default)); Assert.Contains("truncated", ex.Message, StringComparison.OrdinalIgnoreCase); } @@ -488,7 +489,7 @@ public void CentralDirectorySizeShorterThanItsFixedRecordIsRejected(uint cdSize) WriteEocd(zipBytes, eocd, declaredCount: 1, cdSize: cdSize, cdOffset: cdOffset); InvalidDataException ex = Assert.Throws( - () => ZipMemoryIndex.Create(zipBytes, ExcelReaderOptions.Default)); + () => ZipIndex.Create(zipBytes, ExcelReaderOptions.Default)); Assert.Contains("truncated", ex.Message, StringComparison.OrdinalIgnoreCase); } @@ -498,16 +499,268 @@ public void LocalExtraLengthPushingDataPastTheFileEndThrowsInvalidDataOnOpen() byte[] zipBytes = BuildZipWithOneEntry("hello.txt", Zip64Payload, CompressionLevel.NoCompression); BinaryPrimitives.WriteUInt16LittleEndian(zipBytes.AsSpan(28), 0xFFFF); - using ZipMemoryIndex index = ZipMemoryIndex.Create(zipBytes, ExcelReaderOptions.Default); + using ZipIndex index = ZipIndex.Create(zipBytes, ExcelReaderOptions.Default); Assert.True(index.TryGetEntry("hello.txt"u8, out ZipEntryRef entry)); InvalidDataException ex = Assert.Throws( () => index.OpenPart(entry, new DecompressedByteCounter(0))); Assert.Contains("past the end", ex.Message, StringComparison.OrdinalIgnoreCase); } + private static readonly bool[] Modes = [false, true]; + + private static async Task CreatePositionalAsync(byte[] bytes, bool useAsync, CancellationToken ct) + { + ByteSource source = ByteSource.FromStream(new MemoryStream(bytes, writable: false), leaveOpen: false); + return useAsync + ? await ZipIndex.CreateAsync(source, ExcelReaderOptions.Default, ct) + : ZipIndex.Create(source, ExcelReaderOptions.Default); + } + + private static async Task CreateFailsAsync(byte[] bytes, bool useAsync, CancellationToken ct) + { + return await Assert.ThrowsAsync(async () => (await CreatePositionalAsync(bytes, useAsync, ct)).Dispose()); + } + + private static async Task OpenPartPositionalAsync(ZipIndex index, ZipEntryRef entry, bool useAsync, CancellationToken ct) + { + DecompressedByteCounter counter = new(0); + return useAsync ? await index.OpenPartAsync(entry, counter, ct) : index.OpenPart(entry, counter); + } + + private static async Task OpenPartFailsAsync(ZipIndex index, ZipEntryRef entry, bool useAsync, CancellationToken ct) + { + return await Assert.ThrowsAsync(async () => (await OpenPartPositionalAsync(index, entry, useAsync, ct)).Dispose()); + } + + [Fact] + public async Task PositionalValidZip64ArchiveOpensTheStoredEntry() + { + CancellationToken ct = TestContext.Current.CancellationToken; + foreach (bool useAsync in Modes) + { + Zip64Layout zip = BuildStoredZip64(Zip64Payload); + using ZipIndex index = await CreatePositionalAsync(zip.Bytes, useAsync, ct); + Assert.False(index.HasMemory); + Assert.True(index.TryGetEntry("a.txt"u8, out ZipEntryRef entry)); + Assert.Equal(0, entry.LocalHeaderOffset); + Assert.Equal(Zip64Payload.Length, entry.CompressedSize); + Assert.Equal(Zip64Payload.Length, entry.UncompressedSize); + using ZipPart part = await OpenPartPositionalAsync(index, entry, useAsync, ct); + Assert.Equal(Zip64Payload, part.Memory.ToArray()); + } + } + + [Fact] + public async Task PositionalHugeZip64EocdLocatorOffsetThrowsInsteadOfWrapping() + { + CancellationToken ct = TestContext.Current.CancellationToken; + foreach (bool useAsync in Modes) + { + byte[] bytes = new byte[42]; + WriteZip64Locator(bytes, 0, zip64EocdOffset: long.MaxValue - 2); + WriteEocd(bytes, 20, declaredCount: 0xFFFF, cdSize: Zip64SentinelU32, cdOffset: Zip64SentinelU32); + + InvalidDataException ex = await CreateFailsAsync(bytes, useAsync, ct); + Assert.Contains("ZIP64", ex.Message, StringComparison.Ordinal); + } + } + + [Fact] + public async Task PositionalHugeZip64CentralDirectorySizeThrowsInsteadOfWrapping() + { + CancellationToken ct = TestContext.Current.CancellationToken; + foreach (bool useAsync in Modes) + { + byte[] bytes = new byte[98]; + BinaryPrimitives.WriteInt32LittleEndian(bytes.AsSpan(0), 0x06064b50); + BinaryPrimitives.WriteInt64LittleEndian(bytes.AsSpan(32), 1); + BinaryPrimitives.WriteInt64LittleEndian(bytes.AsSpan(40), long.MaxValue - 2); + BinaryPrimitives.WriteInt64LittleEndian(bytes.AsSpan(48), 5); + WriteZip64Locator(bytes, 56, zip64EocdOffset: 0); + WriteEocd(bytes, 76, declaredCount: 0xFFFF, cdSize: Zip64SentinelU32, cdOffset: Zip64SentinelU32); + + InvalidDataException ex = await CreateFailsAsync(bytes, useAsync, ct); + Assert.Contains("central directory", ex.Message, StringComparison.OrdinalIgnoreCase); + } + } + + [Fact] + public async Task PositionalNegativeZip64SizesThrowInvalidDataException() + { + CancellationToken ct = TestContext.Current.CancellationToken; + foreach (bool useAsync in Modes) + { + byte[] uncompressed = BuildCdWithZip64Field(compressedSize: 5, uncompressedSize: Zip64SentinelU32, zip64Value: -1); + Assert.Contains("negative", (await CreateFailsAsync(uncompressed, useAsync, ct)).Message, StringComparison.OrdinalIgnoreCase); + + byte[] compressed = BuildCdWithZip64Field(compressedSize: Zip64SentinelU32, uncompressedSize: 5, zip64Value: -1); + Assert.Contains("negative", (await CreateFailsAsync(compressed, useAsync, ct)).Message, StringComparison.OrdinalIgnoreCase); + } + } + + [Theory] + [InlineData(long.MaxValue)] + [InlineData(long.MaxValue - 29)] + [InlineData(long.MaxValue - 30)] + [InlineData((long)int.MaxValue + 1)] + [InlineData(int.MaxValue)] + public async Task PositionalHugeZip64LocalHeaderOffsetThrowsInvalidDataOnOpen(long localOffset) + { + CancellationToken ct = TestContext.Current.CancellationToken; + foreach (bool useAsync in Modes) + { + Zip64Layout zip = BuildStoredZip64(Zip64Payload, localOffset: localOffset); + + using ZipIndex index = await CreatePositionalAsync(zip.Bytes, useAsync, ct); + Assert.True(index.TryGetEntry("a.txt"u8, out ZipEntryRef entry)); + Assert.Equal(localOffset, entry.LocalHeaderOffset); + InvalidDataException ex = await OpenPartFailsAsync(index, entry, useAsync, ct); + Assert.Contains("local file header", ex.Message, StringComparison.OrdinalIgnoreCase); + } + } + + [Fact] + public async Task PositionalLocalHeaderOffsetJustInsideTheFileEndThrowsInvalidDataOnOpen() + { + CancellationToken ct = TestContext.Current.CancellationToken; + foreach (bool useAsync in Modes) + { + Zip64Layout probe = BuildStoredZip64(Zip64Payload); + Zip64Layout zip = BuildStoredZip64(Zip64Payload, localOffset: probe.Bytes.Length - 29); + + using ZipIndex index = await CreatePositionalAsync(zip.Bytes, useAsync, ct); + Assert.True(index.TryGetEntry("a.txt"u8, out ZipEntryRef entry)); + InvalidDataException ex = await OpenPartFailsAsync(index, entry, useAsync, ct); + Assert.Contains("out of range", ex.Message, StringComparison.OrdinalIgnoreCase); + } + } + + [Theory] + [InlineData(long.MaxValue)] + [InlineData(long.MaxValue - 50)] + [InlineData((long)int.MaxValue + 1)] + [InlineData(int.MaxValue)] + public async Task PositionalHugeZip64CompressedSizeThrowsInvalidDataOnOpen(long compressedSize) + { + CancellationToken ct = TestContext.Current.CancellationToken; + foreach (bool useAsync in Modes) + { + Zip64Layout zip = BuildStoredZip64(Zip64Payload, compressedSize: compressedSize); + + using ZipIndex index = await CreatePositionalAsync(zip.Bytes, useAsync, ct); + Assert.True(index.TryGetEntry("a.txt"u8, out ZipEntryRef entry)); + Assert.Equal(compressedSize, entry.CompressedSize); + InvalidDataException ex = await OpenPartFailsAsync(index, entry, useAsync, ct); + Assert.Contains("past the end", ex.Message, StringComparison.OrdinalIgnoreCase); + if (useAsync) + { + await Assert.ThrowsAsync( + async () => (await index.OpenEntryStreamAsync(entry, new DecompressedByteCounter(0), ExcelReaderOptions.Default, ct)).Dispose()); + } + else + { + Assert.Throws( + () => index.OpenEntryStream(entry, new DecompressedByteCounter(0), ExcelReaderOptions.Default)); + } + } + } + + [Fact] + public async Task PositionalCompressedSizeEndingExactlyAtTheFileEndIsAcceptedAndOneMoreIsRejected() + { + CancellationToken ct = TestContext.Current.CancellationToken; + foreach (bool useAsync in Modes) + { + Zip64Layout probe = BuildStoredZip64(Zip64Payload); + long exact = probe.Bytes.Length - probe.DataOffset; + + Zip64Layout fits = BuildStoredZip64(Zip64Payload, compressedSize: exact, uncompressedSize: exact); + using (ZipIndex index = await CreatePositionalAsync(fits.Bytes, useAsync, ct)) + { + Assert.True(index.TryGetEntry("a.txt"u8, out ZipEntryRef entry)); + using ZipPart part = await OpenPartPositionalAsync(index, entry, useAsync, ct); + Assert.Equal(exact, part.Memory.Length); + } + + Zip64Layout overruns = BuildStoredZip64(Zip64Payload, compressedSize: exact + 1); + using (ZipIndex index = await CreatePositionalAsync(overruns.Bytes, useAsync, ct)) + { + Assert.True(index.TryGetEntry("a.txt"u8, out ZipEntryRef entry)); + await OpenPartFailsAsync(index, entry, useAsync, ct); + } + } + } + + [Theory] + [InlineData(0x8000000000000000UL)] + [InlineData(ulong.MaxValue)] + public async Task PositionalZip64LocalHeaderOffsetAboveLongMaxIsRejectedAsNegative(ulong localOffset) + { + CancellationToken ct = TestContext.Current.CancellationToken; + foreach (bool useAsync in Modes) + { + Zip64Layout zip = BuildStoredZip64(Zip64Payload, localOffset: unchecked((long)localOffset)); + + InvalidDataException ex = await CreateFailsAsync(zip.Bytes, useAsync, ct); + Assert.Contains("negative", ex.Message, StringComparison.OrdinalIgnoreCase); + } + } + + [Theory] + [InlineData(0x8000000000000000UL)] + [InlineData(ulong.MaxValue)] + public async Task PositionalZip64LocatorOffsetAboveLongMaxIsRejectedNotIgnored(ulong zip64EocdOffset) + { + CancellationToken ct = TestContext.Current.CancellationToken; + foreach (bool useAsync in Modes) + { + Zip64Layout zip = BuildStoredZip64(Zip64Payload); + BinaryPrimitives.WriteUInt64LittleEndian(zip.Bytes.AsSpan(zip.LocatorOffset + 8), zip64EocdOffset); + + InvalidDataException ex = await CreateFailsAsync(zip.Bytes, useAsync, ct); + Assert.Contains("ZIP64", ex.Message, StringComparison.Ordinal); + } + } + + [Theory] + [InlineData(long.MaxValue, 1UL)] + [InlineData(long.MaxValue, (ulong)long.MaxValue)] + [InlineData(1L, (ulong)long.MaxValue)] + [InlineData(0L, ulong.MaxValue)] + [InlineData(-1L, 0UL)] + [InlineData(long.MinValue, 100UL)] + [InlineData(0L, 0x8000000000000000UL)] + public async Task PositionalZip64CentralDirectoryBoundsThatWouldOverflowAreRejected(long cdOffset, ulong cdSize) + { + CancellationToken ct = TestContext.Current.CancellationToken; + foreach (bool useAsync in Modes) + { + Zip64Layout zip = BuildStoredZip64(Zip64Payload); + BinaryPrimitives.WriteUInt64LittleEndian(zip.Bytes.AsSpan(zip.Zip64EocdOffset + 40), cdSize); + BinaryPrimitives.WriteInt64LittleEndian(zip.Bytes.AsSpan(zip.Zip64EocdOffset + 48), cdOffset); + + InvalidDataException ex = await CreateFailsAsync(zip.Bytes, useAsync, ct); + Assert.Contains("central directory", ex.Message, StringComparison.OrdinalIgnoreCase); + } + } + + [Fact] + public async Task PositionalZip64CentralDirectorySizeOneByteLongerThanItsRecordsIsRejected() + { + CancellationToken ct = TestContext.Current.CancellationToken; + foreach (bool useAsync in Modes) + { + Zip64Layout zip = BuildStoredZip64(Zip64Payload); + long cdSize = BinaryPrimitives.ReadInt64LittleEndian(zip.Bytes.AsSpan(zip.Zip64EocdOffset + 40)); + BinaryPrimitives.WriteInt64LittleEndian(zip.Bytes.AsSpan(zip.Zip64EocdOffset + 40), cdSize + 1); + + await CreateFailsAsync(zip.Bytes, useAsync, ct); + } + } + private readonly record struct Zip64Layout(byte[] Bytes, int DataOffset, int Zip64EocdOffset, int LocatorOffset); - private static Zip64Layout BuildStoredZip64(byte[] payload, long? localOffset = null, long? compressedSize = null) + private static Zip64Layout BuildStoredZip64(byte[] payload, long? localOffset = null, long? compressedSize = null, long? uncompressedSize = null) { byte[] name = "a.txt"u8.ToArray(); uint crc = Crc32(payload); @@ -544,7 +797,7 @@ private static Zip64Layout BuildStoredZip64(byte[] payload, long? localOffset = BinaryPrimitives.WriteUInt32LittleEndian(cd[42..], Zip64SentinelU32); name.CopyTo(cd[CentralDirectoryFixedSize..]); WriteZip64Extra(cd[(CentralDirectoryFixedSize + name.Length)..], - payload.Length, compressedSize ?? payload.Length, localOffset ?? 0); + uncompressedSize ?? payload.Length, compressedSize ?? payload.Length, localOffset ?? 0); Span z64 = s[zip64EocdOffset..]; BinaryPrimitives.WriteInt32LittleEndian(z64, 0x06064b50); @@ -588,21 +841,6 @@ private static uint Crc32(ReadOnlySpan data) return ~crc; } - [Fact] - public void ZipEntryBytesReadThrowsInvalidDataWhenEntryUnderDelivers() - { - using MemoryStream built = WorkbookBuilder.Build("""1"""); - byte[] zipBytes = built.ToArray(); - uint realLength = ReadDeclaredUncompressedSize(zipBytes, "xl/workbook.xml"); - PatchCentralDirectoryUInt32(zipBytes, "xl/workbook.xml", fieldOffset: 24, realLength + 64); - - using var ms = new MemoryStream(zipBytes); - using var zip = new ZipArchive(ms, ZipArchiveMode.Read); - InvalidDataException ex = Assert.Throws( - () => ZipEntryBytes.Read(zip, "xl/workbook.xml", new DecompressedByteCounter(0))); - Assert.Contains("less data", ex.Message, StringComparison.OrdinalIgnoreCase); - } - [Fact] public void ZipArchiveEntryOpenSilentlyTruncatesAtDeclaredLengthUnderOverDelivery() { @@ -631,8 +869,10 @@ private static byte[] ReadViaZipArchive(byte[] zipBytes, string entryName) { using var ms = new MemoryStream(zipBytes); using var zip = new ZipArchive(ms, ZipArchiveMode.Read); - using ZipPart part = ZipEntryBytes.Read(zip, entryName, new DecompressedByteCounter(0)); - return part.Memory.ToArray(); + using Stream entry = zip.GetEntry(entryName)!.Open(); + using var copy = new MemoryStream(); + entry.CopyTo(copy); + return copy.ToArray(); } private static byte[] BuildZipWithOneEntry(string entryName, byte[] payload, CompressionLevel level) diff --git a/tests/ExcelReader.Tests/TestUtils.cs b/tests/ExcelReader.Tests/TestUtils.cs index 37e41eb6..7e6653b8 100644 --- a/tests/ExcelReader.Tests/TestUtils.cs +++ b/tests/ExcelReader.Tests/TestUtils.cs @@ -145,6 +145,82 @@ protected override void Dispose(bool disposing) } } + internal sealed class TrickleStream : Stream + { + private const int MaxPerRead = 7; + + private readonly byte[] _bytes; + private long _position; + private bool _disposed; + + internal TrickleStream(byte[] bytes) + { + _bytes = bytes; + } + + internal Action? OnRead { get; set; } + + public override bool CanRead => !_disposed; + + public override bool CanSeek => !_disposed; + + public override bool CanWrite => false; + + public override long Length => _bytes.Length; + + public override long Position + { + get => _position; + set => _position = value; + } + + public override int Read(byte[] buffer, int offset, int count) + { + return Read(buffer.AsSpan(offset, count)); + } + + public override int Read(Span buffer) + { + ObjectDisposedException.ThrowIf(_disposed, this); + OnRead?.Invoke(); + int available = (int)Math.Max(0, Math.Min(_bytes.Length - _position, Math.Min(buffer.Length, MaxPerRead))); + _bytes.AsSpan((int)_position, available).CopyTo(buffer); + _position += available; + return available; + } + + public override long Seek(long offset, SeekOrigin origin) + { + _position = origin switch + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => _position + offset, + _ => _bytes.Length + offset, + }; + return _position; + } + + public override void Flush() + { + } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotSupportedException(); + } + + protected override void Dispose(bool disposing) + { + _disposed = true; + base.Dispose(disposing); + } + } + internal sealed class TrackingStream : MemoryStream { internal TrackingStream()