True streaming packages: format entities as Packages (#35)

README.md

@@ -103,6 +103,8 @@ Optional: if `dotnet restore` feels slow because of publish RIDs, you can tempor
 
 CI: GitHub Actions (`.github/workflows/ci.yml`) builds on .NET 10 and runs `src/test.sh`.
 
+Architecture note: binary/OpenXML reading is organized as **streaming packages** (emit a complete local format unit as soon as it is recognized) so a renderer can progress while data is still arriving — see [`docs/streaming-packages.md`](docs/streaming-packages.md).
+
 ## Build on MacOS
 ```bash
 dotnet publish -r osx-x64 src/Exceltk/Exceltk.csproj -c Release

docs/streaming-packages.md

@@ -0,0 +1,44 @@
+# Streaming package / progressive render
+
+ExcelTk's binary and OpenXML readers are built around **packages**: complete local
+format units that a parser can emit as soon as enough input has arrived.
+
+## Mental model
+
+```
+network / file Stream
+        │
+        ▼
+  PackageParser   ── recognizes a finished local unit ──►  Package
+        │                                                    │
+        │ (continue)                                         ▼
+        │                                              Renderer / DataSet builder
+```
+
+Examples of a “finished local unit”:
+
+| Format | Package | Completeness boundary |
+| --- | --- | --- |
+| BIFF (.xls) | `XlsBiffRecord` (: `BinaryPackage`) | one record header (`type`+`size`) + body |
+| OpenXML sheet | `XmlCellPackage` / `XmlRowPackage` | finished `</c>` / `</row>` |
+| OpenXML | `XmlMergePackage` / `XmlHyperlinkPackage` | finished empty element |
+
+A progressive renderer can subscribe to `IPackageParser<T>.Parse()` (or
+`IPullPackageParser.TryRead`) and paint each package without waiting for EOF.
+
+## What this codebase does today
+
+- **Binary**: `BinaryPackageParser` slices **one BIFF record** into an owned byte
+  buffer per emit. OLE sector assembly still materializes the workbook stream
+  (compound-file seeks); record emission itself is one-package-at-a-time.
+- **OpenXML**: `ZipWorker` keeps the ZIP open and **streams entry inflate**
+  (`GetInputStream`) instead of extracting the whole archive to disk.
+  `XmlPackageParser` emits row/cell/merge/hyperlink packages from `XmlReader`.
+- **Batch path**: markdown/json/tex still build a `DataSet` by consuming the
+  package stream (convenient, not required for progressive UI).
+
+## Non-seekable network ZIP
+
+Fully forward-only OPC (single `ZipInputStream` pass over a non-seekable network
+body) is a follow-up: sheet parts are not contiguous with workbook.xml. Seekable
+streams (files, buffered downloads) are supported via `ZipFile` entry streaming.

src/Exceltk/Reader/Binary/XlsBiffRecord.cs

@@ -1,12 +1,13 @@
 using System;
+using Exceltk.Reader.Package;
 
 namespace Exceltk.Reader.Binary {
     /// <summary>
     /// BIFF:Binary Interchange File Format
-    /// Represents basic BIFF record
-    /// Base class for all BIFF record types
+    /// Represents basic BIFF record — itself a <see cref="BinaryPackage"/> so a streaming
+    /// parser can emit it as soon as one record (type+size+body) is complete.
     /// </summary>
-    internal class XlsBiffRecord {
+    internal class XlsBiffRecord : BinaryPackage {
         protected readonly ExcelBinaryReader reader;
         protected byte[] m_bytes;
         protected int m_readoffset;

src/Exceltk/Reader/ExcelBinaryReader.cs

@@ -131,9 +131,6 @@ public void Close() {
 
 
 
-        private static XlsBiffRecord RecordOf(BinaryPackage package) {
-            return package==null ? null : package.Record;
-        }
 
         #region Private methods
 
@@ -157,7 +154,7 @@ private bool Read() {
 
         private int findFirstDataCellOffset(int startOffset) {
             //seek to the first dbcell record
-            XlsBiffRecord record=RecordOf(m_stream.ReadAt(startOffset));
+            XlsBiffRecord record=m_stream.ReadAt(startOffset);
             while (!(record is XlsBiffDbCell)) {
                 if (m_stream.Position >= m_stream.Size) {
                     return -1;
@@ -168,7 +165,7 @@ private int findFirstDataCellOffset(int startOffset) {
                 }
 
                 try {
-                    record=RecordOf(m_stream.Read());
+                    record=m_stream.Read();
                 } catch {
                     return -2;
                 }
@@ -180,7 +177,7 @@ private int findFirstDataCellOffset(int startOffset) {
             int offs=startCell.RowAddress;
 
             do {
-                row=RecordOf(m_stream.ReadAt(offs)) as XlsBiffRow;
+                row=m_stream.ReadAt(offs) as XlsBiffRow;
                 if (row==null){
                     break;
                 }
@@ -221,7 +218,7 @@ private void readWorkBookGlobals() {
 
             m_stream.Seek(0, SeekOrigin.Begin);
 
-            XlsBiffRecord rec=RecordOf(m_stream.Read());
+            XlsBiffRecord rec=m_stream.Read();
             var bof=rec as XlsBiffBOF;
 
             if (bof==null||bof.Type!=BIFFTYPE.WorkbookGlobals) {
@@ -234,7 +231,7 @@ private void readWorkBookGlobals() {
             m_version=bof.Version;
             m_sheets=new List<XlsWorksheet>();
 
-            while (null!=(rec=RecordOf(m_stream.Read()))) {
+            while (null!=(rec=m_stream.Read())) {
                 switch (rec.ID) {
                     case BIFFRECORDTYPE.INTERFACEHDR:
                         m_globals.InterfaceHdr=(XlsBiffInterfaceHdr)rec;
@@ -328,20 +325,20 @@ private bool readWorkSheetGlobals(XlsWorksheet sheet, out XlsBiffIndex idx, out
             m_stream.Seek((int)sheet.DataOffset, SeekOrigin.Begin);
 
             // Read BOF
-            var bof=RecordOf(m_stream.Read()) as XlsBiffBOF;
+            var bof=m_stream.Read() as XlsBiffBOF;
             if (bof==null||bof.Type!=BIFFTYPE.Worksheet) {
                 return false;
             }
 
             // Read Index
-            XlsBiffRecord rec=RecordOf(m_stream.Read());
+            XlsBiffRecord rec=m_stream.Read();
             if (rec==null)
                 return false;
             if (rec is XlsBiffIndex) {
                 idx=rec as XlsBiffIndex;
             } else if (rec is XlsBiffUncalced) {
                 // Sometimes this come before the index...
-                idx=RecordOf(m_stream.Read()) as XlsBiffIndex;
+                idx=m_stream.Read() as XlsBiffIndex;
             }
 
             if (idx!=null) {
@@ -353,7 +350,7 @@ private bool readWorkSheetGlobals(XlsWorksheet sheet, out XlsBiffIndex idx, out
             XlsBiffDimensions dims=null;
 
             do {
-                trec=RecordOf(m_stream.Read());
+                trec=m_stream.Read();
                 if (trec.ID==BIFFRECORDTYPE.DIMENSIONS) {
                     dims=(XlsBiffDimensions)trec;
                     break;
@@ -370,7 +367,7 @@ private bool readWorkSheetGlobals(XlsWorksheet sheet, out XlsBiffIndex idx, out
             while (rowRecord==null) {
                 if (m_stream.Position>=m_stream.Size)
                     break;
-                XlsBiffRecord thisRec=RecordOf(m_stream.Read());
+                XlsBiffRecord thisRec=m_stream.Read();
 
                 if (thisRec is XlsBiffEOF)
                     break;
@@ -409,7 +406,7 @@ private bool readWorkSheetGlobals(XlsWorksheet sheet, out XlsBiffIndex idx, out
             while (true) {
                 if (m_stream.Position>=m_stream.Size)
                     break;
-                XlsBiffRecord thisRecord=RecordOf(m_stream.Read());
+                XlsBiffRecord thisRecord=m_stream.Read();
 
                 if (thisRecord is XlsBiffEOF) {
                     break;
@@ -434,7 +431,7 @@ private void DumpBiffRecords() {
             int startPos=m_stream.Position;
 
             do {
-                rec=RecordOf(m_stream.Read());
+                rec=m_stream.Read();
             } while (rec!=null&&m_stream.Position<m_stream.Size);
 
             m_stream.Seek(startPos, SeekOrigin.Begin);
@@ -444,7 +441,7 @@ private bool readWorkSheetRow() {
             m_cellsValues=new XlsCell[m_maxCol];
 
             while (m_cellOffset<m_stream.Size) {
-                XlsBiffRecord rec=RecordOf(m_stream.ReadAt(m_cellOffset));
+                XlsBiffRecord rec=m_stream.ReadAt(m_cellOffset);
 
                 m_cellOffset+=rec.Size;
 
@@ -698,7 +695,7 @@ private bool moveToNextRecordNoIndex() {
                         return false;
                     }
 
-                    XlsBiffRecord record=RecordOf(m_stream.Read());
+                    XlsBiffRecord record=m_stream.Read();
                     if (record is XlsBiffEOF) {
                         return false;
                     }
@@ -716,7 +713,7 @@ private bool moveToNextRecordNoIndex() {
                     return false;
                 }
 
-                XlsBiffRecord record=RecordOf(m_stream.Read());
+                XlsBiffRecord record=m_stream.Read();
                 if (record is XlsBiffEOF) {
                     return false;
                 }

src/Exceltk/Reader/ExcelOpenXmlReader.cs

@@ -5,7 +5,6 @@
 using System.IO;
 using System.Xml;
 using System.Text;
-using Exceltk.Reader.Package;
 using Exceltk.Reader.Parser;
 using Exceltk.Reader.Xml;
 

src/Exceltk/Reader/Package/IPackage.cs

@@ -1,8 +1,21 @@
 namespace Exceltk.Reader.Package {
     /// <summary>
-    /// A parsed semantic unit emitted by a streaming package parser.
-    /// Not a file container — binary records and XML elements are packages.
+    /// A complete local format unit that a streaming parser can emit as soon as
+    /// enough bytes/tokens arrive (e.g. one BIFF record, one XML cell/row).
+    /// Suitable for progressive render over a network transfer.
     /// </summary>
     public interface IPackage {
     }
+
+    /// <summary>
+    /// Binary (BIFF) package base. Concrete record types in <c>Reader.Binary</c> inherit this.
+    /// </summary>
+    public abstract class BinaryPackage : IPackage {
+    }
+
+    /// <summary>
+    /// OpenXML / worksheet XML package base. Concrete units live in <c>Reader.Xml</c>.
+    /// </summary>
+    public abstract class XmlPackage : IPackage {
+    }
 }

src/Exceltk/Reader/Parser/BinaryPackageParser.cs

@@ -6,10 +6,12 @@
 
 namespace Exceltk.Reader.Parser {
     /// <summary>
-    /// Streams <see cref="BinaryPackage"/> entities (one BIFF record each) from a workbook stream.
+    /// Streams BIFF <see cref="BinaryPackage"/> entities (each <see cref="XlsBiffRecord"/>)
+    /// from a workbook stream. Emits a package as soon as one record's header+body is complete;
+    /// each package owns an isolated byte slice so a renderer need not retain the full workbook.
     /// </summary>
-    internal sealed class BinaryPackageParser : XlsStream, IPackageParser<BinaryPackage> {
-        private readonly byte[] m_bytes;
+    internal sealed class BinaryPackageParser : XlsStream, IPullPackageParser<BinaryPackage> {
+        private readonly byte[] m_workbookBytes;
         private readonly int m_size;
         private readonly ExcelBinaryReader m_reader;
         private int m_offset;
@@ -18,8 +20,10 @@ public BinaryPackageParser(XlsHeader hdr, uint streamStart, bool isMini, XlsRoot
             ExcelBinaryReader reader)
             : base(hdr, streamStart, isMini, rootDir) {
             m_reader = reader;
-            m_bytes = base.ReadStream();
-            m_size = m_bytes.Length;
+            // OLE compound storage still needs sector assembly to reach the workbook stream;
+            // once positioned, we only slice one record at a time into each package.
+            m_workbookBytes = base.ReadStream();
+            m_size = m_workbookBytes.Length;
             m_offset = 0;
         }
 
@@ -60,50 +64,67 @@ public void Seek(int offset, SeekOrigin origin) {
         }
 
         /// <summary>
-        /// Read the next BIFF record as a <see cref="BinaryPackage"/> and advance the cursor.
+        /// Read the next complete BIFF record package (local format unit) and advance.
         /// </summary>
-        public BinaryPackage Read() {
-            if ((uint)m_offset >= m_bytes.Length) {
+        public XlsBiffRecord Read() {
+            BinaryPackage package;
+            if (!TryRead(out package)) {
                 return null;
             }
+            return (XlsBiffRecord)package;
+        }
 
-            XlsBiffRecord rec = XlsBiffRecord.GetRecord(m_bytes, (uint)m_offset, m_reader);
-            m_offset += rec.Size;
+        /// <summary>
+        /// Peek/build a record package at an absolute workbook-stream offset (no cursor move).
+        /// Still allocates a one-record slice — suitable for index/DBCELL jumps.
+        /// </summary>
+        public XlsBiffRecord ReadAt(int offset) {
+            if ((uint)offset >= m_workbookBytes.Length || offset + 4 > m_size) {
+                return null;
+            }
 
-            if (m_offset > m_size) {
+            ushort recordSize = BitConverter.ToUInt16(m_workbookBytes, offset + 2);
+            int size = 4 + recordSize;
+            if (m_reader.ReadOption == ReadOption.Strict && offset + size > m_size) {
                 return null;
             }
+            if (offset + size > m_size) {
+                size = m_size - offset;
+            }
 
-            return new BinaryPackage(rec);
+            byte[] slice = SliceRecord(offset, size);
+            return XlsBiffRecord.GetRecord(slice, 0, m_reader);
         }
 
-        /// <summary>
-        /// Read a record package at an absolute offset without moving the cursor.
-        /// </summary>
-        public BinaryPackage ReadAt(int offset) {
-            if ((uint)offset >= m_bytes.Length) {
-                return null;
+        public bool TryRead(out BinaryPackage package) {
+            package = null;
+            if ((uint)m_offset >= m_workbookBytes.Length || m_offset + 4 > m_size) {
+                return false;
             }
 
-            XlsBiffRecord rec = XlsBiffRecord.GetRecord(m_bytes, (uint)offset, m_reader);
-
-            if (m_reader.ReadOption == ReadOption.Strict) {
-                if (offset + rec.Size > m_size) {
-                    return null;
-                }
+            ushort recordSize = BitConverter.ToUInt16(m_workbookBytes, m_offset + 2);
+            int size = 4 + recordSize;
+            if (m_offset + size > m_size) {
+                return false;
             }
 
-            return new BinaryPackage(rec);
+            byte[] slice = SliceRecord(m_offset, size);
+            m_offset += size;
+            package = XlsBiffRecord.GetRecord(slice, 0, m_reader);
+            return package != null;
         }
 
-        /// <summary>
-        /// Stream all remaining packages from the current cursor to the end.
-        /// </summary>
         public IEnumerable<BinaryPackage> Parse() {
             BinaryPackage package;
-            while ((package = Read()) != null) {
+            while (TryRead(out package)) {
                 yield return package;
             }
         }
+
+        private byte[] SliceRecord(int offset, int size) {
+            var slice = new byte[size];
+            Buffer.BlockCopy(m_workbookBytes, offset, slice, 0, size);
+            return slice;
+        }
     }
 }