diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index c08e0b7..0000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,38 +0,0 @@ -# CodeQL static analysis of the C# code. Findings land in the repository's Security tab. Runs on every -# push and pull request to main and weekly, so a newly published query also covers old code. -name: CodeQL - -on: - push: - branches: [main] - pull_request: - branches: [main] - schedule: - - cron: '15 7 * * 1' - -permissions: - contents: read - -jobs: - analyze: - runs-on: ubuntu-latest - permissions: - security-events: write # upload the results - contents: read - steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - - uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4 - with: - dotnet-version: | - 8.0.x - 10.0.x - - uses: github/codeql-action/init@9f759ee644a3e7c15c1390abf49868036c00067b # v3 - with: - languages: csharp - build-mode: manual - # The analyzer and generator projects are built too, so their code is covered along with the library. - - name: Build - run: dotnet build -c Release - - uses: github/codeql-action/analyze@9f759ee644a3e7c15c1390abf49868036c00067b # v3 - with: - category: csharp diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b2198a..a94c704 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ All notable changes to SheetStream. Versions follow [Semantic Versioning](https: - **Edit small existing workbooks**: `ExcelPackage.Edit(path)` (in place, atomic replace) or `Edit(input, output)`, then `Worksheets.Edit(name)` loads the sheet into `Cells[...]`: read values, change cells, add rows, `Cells.Existing`, `Cells.Dimension`, `ExcelCell.Clear()`. Untouched cells, rows, formulas (shared and array formulas included), charts, tables and other sheets are written back unchanged. Guarded by `PackageOptions.MaxEditableCells` (200,000 by default); large files keep using streaming or templates. - **Trimming and Native AOT**: the `net8.0` build is `IsTrimmable` and `IsAotCompatible`. A bundled **source generator** turns `[SheetRow] partial` types into compile-time mappers (`ISheetRowMapper`, `SheetMappers`), so `LoadFromCollection`, `LoadFromAsyncEnumerable` and `ExcelReader.Read` work without reflection or `Expression.Compile`; new overloads take a mapper explicitly. Reflection stays the fallback for unmarked types and is now annotated for the trimmer. - **Roslyn analyzer** shipped inside the package: **SS0001** layout setting after rows were written, **SS0002** `Row()` and `Cells[]` mixed on one sheet, **SS0003** rows written out of order. Warnings at design time instead of exceptions at run time. +- **Pivot tables**: `Worksheet.AddPivotTable` over a table or range written earlier, with row, column, filter and data fields (Sum, Count, Average, Max, Min and more), styles and grand totals. The cache is marked refresh-on-open, so Excel computes the pivot when the file opens; verified against Excel 16 through COM automation (`tests/excel-check-pivot.ps1`). +- **Cached formula results**: `Worksheet.CacheFormulaResults()` opts a sheet into a small evaluator (`+ - * / %`, `SUM`, `AVERAGE`, `COUNT`, `COUNTA`, `MIN`, `MAX`, `ROUND`, `ABS` over cells already written) so viewers that do not calculate show totals instead of blanks. Anything it cannot prove is written without a value, as before. +- **Async reading** (.NET 8+): `ExcelReader.OpenAsync`, `ReadRowsAsync`, `ReadAsync` and `ReadDataTableAsync`, with cancellation between rows; non-seekable uploads are buffered asynchronously and spill to a temp file above 64 MB. - **Reader**: `ReadMergedRanges` and `ReadHyperlinks` (issue #6) read a sheet's merged cells and links, external URLs resolved from the sheet relationships, without parsing the rows. - **Benchmarks**: 1,000,000-row and 1,000 × 10,000-column write scenarios, and read benchmarks against MiniExcel, ExcelDataReader, ClosedXML, EPPlus and NPOI. - **Web sample** (`samples/SheetStream.Sample.Web`): a minimal API with a download button, a 50,000-row streamed export and a `Cells[...]` report with charts; double-click `run.cmd`. A [DotNetFiddle](https://dotnetfiddle.net/BR59ju) runs the quick start in the browser. diff --git a/README.md b/README.md index 5319cd7..1875961 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,19 @@ sheet.AddPicture("logo.png", row: 1, column: 8, widthPixels: 160); Chart types: column, bar (clustered, stacked, 100%), line, line with markers, stacked line, area, stacked area, pie, doughnut, scatter and scatter with lines. Pictures: PNG, JPEG and GIF. +### Pivot tables + +```csharp +var loaded = data.LoadFromCollection(orders, new LoadOptions { TableName = "Orders" }); +var pivot = workbook.Worksheets.Add("Pivot").AddPivotTable("SalesByRegion", loaded.Table!, row: 3, column: 1); +pivot.AddRowField("Region"); +pivot.AddColumnField("Category"); +pivot.AddDataField("Amount", PivotFunction.Sum, numberFormat: "$#,##0.00"); +pivot.AddDataField("OrderId", PivotFunction.Count, displayName: "Orders"); +``` + +The pivot sits over a table or range written earlier (`AddPivotTable("P", "Data", loaded.Range, 3, 1, loaded.Headers)` for a plain range) and is marked refresh-on-open, so Excel computes it when the file opens; the output is checked against real Excel in `tests/excel-check-pivot.ps1`. Viewers without a pivot engine show only the headers until Excel has opened and saved the file. Calculated fields, grouping and slicers are not supported. See [Pivot tables](docs/pivot-tables.md). + ### Links, comments, rich text and formulas ```csharp @@ -142,6 +155,16 @@ sheet.AddComment(2, 3, "Checked by finance", author: "Glenn"); workbook.DefinedNames.Add("TaxRate", "'Settings'!B2"); ``` +### Formula results for viewers that do not calculate + +```csharp +sheet.CacheFormulaResults(); // Before the first row; about 8 bytes per numeric cell +sheet.Row().WriteCell("Total").WriteFormula("SUM(C2:C101)"); // Written with its result +sheet.Row().WriteCell("Avg").WriteFormula("ROUND(AVERAGE(C2:C101), 2)"); +``` + +File previews, Quick Look, some mobile viewers and `ExcelReader` show formula cells blank unless the file carries a cached result. Opt in per sheet and SheetStream evaluates `+ - * / %`, `SUM`, `AVERAGE`, `COUNT`, `COUNTA`, `MIN`, `MAX`, `ROUND` and `ABS` over cells already written on that sheet. Anything else (other sheets, `IF`, forward references, names) is written as before, with no value and Excel calculating on open: never a wrong value. + ### Layout, printing and protection ```csharp @@ -254,6 +277,15 @@ foreach (var link in reader.ReadHyperlinks("Report")) // Url for web/mai Merged ranges and hyperlinks are read from the sheet's trailer only, so the rows are skipped and the call is instant even on a million-row sheet. +```csharp +// Async reading (.NET 8+): buffer an upload without blocking, then stream rows or objects. +using var reader = await ExcelReader.OpenAsync(Request.Body, cancellationToken: ct); // Non-seekable: copied async, >64 MB spills to a temp file +await foreach (var row in reader.ReadRowsAsync("Products", ct)) // Cancellation checked between rows + Console.WriteLine($"{row.GetString(0)} = {row.GetDouble(2)}"); +await foreach (var p in reader.ReadAsync("Products", ct)) db.Products.Add(p); // Or ReadAsync(sheet, Product.SheetMapper, ct) for AOT +var table = await reader.ReadDataTableAsync("Products", firstRowIsHeader: true, ct); +``` + ### Design-time analyzer The package ships a Roslyn analyzer, so the IDE warns before you run: **SS0001** layout (`FreezePanes`, `Column`, `TabColor`...) set after rows were written, **SS0002** `Row()` and `Cells[]` mixed on one sheet, **SS0003** rows written out of order. It only reports what it can prove inside one method, so a warning means the code will throw. Nothing to install, nothing added to your output. Details, before/after snippets and how to suppress: [docs/analyzers.md](docs/analyzers.md). @@ -268,10 +300,10 @@ The package ships a Roslyn analyzer, so the IDE warns before you run: **SS0001** |---|---| | `ExcelPackage` | `Create(path)`, `new ExcelPackage(stream)`, `CreateBytes(build)`, `Save()`, `Dispose()` | | `Workbook` | `Worksheets.Add`, `Styles`, `DefinedNames`, `Properties`, `Protect` | -| `Worksheet` | `Row()`, `Cells[r, c]`, `Column` / `Columns`, `AutoFitColumns`, `FreezePanes`, `AutoFilter`, `MergeCells`, `AddTable`, `AddChart`, `AddPicture`, `AddConditionalFormatting`, `AddDataValidation`, `AddHyperlink`, `AddComment`, `Protect`, `PageSetup`, `TabColor`, `Visibility`, `LoadFromCollection` / `DataTable` / `DataReader` | +| `Worksheet` | `Row()`, `Cells[r, c]`, `Column` / `Columns`, `AutoFitColumns`, `FreezePanes`, `AutoFilter`, `MergeCells`, `AddTable`, `AddChart`, `AddPicture`, `AddPivotTable`, `AddConditionalFormatting`, `AddDataValidation`, `AddHyperlink`, `AddComment`, `CacheFormulaResults`, `Protect`, `PageSetup`, `TabColor`, `Visibility`, `LoadFromCollection` / `DataTable` / `DataReader` / `AsyncEnumerable` | | `RowWriter` | `WriteCell` (string, numbers, bool, dates, TimeSpan, object), `WriteCells`, `WriteFormula` (with or without cached value), `WriteRichText`, `WriteHyperlink`, `WriteInternalLink`, `WriteInlineString`, `WriteEmpty`, `Skip` | | `StyleSheet` | `Create` (font, fill, border, number format, alignment, wrap, indent, rotation, shrink, protection), `CreateDifferential`, `Hyperlink` | -| `ExcelReader` | `Open`, `SheetNames`, `ReadRows`, `Read`, `ReadDataTable`, `ReadMergedRanges`, `ReadHyperlinks` | +| `ExcelReader` | `Open` / `OpenAsync`, `SheetNames`, `ReadRows` / `ReadRowsAsync`, `Read` / `ReadAsync`, `ReadDataTable` / `ReadDataTableAsync`, `ReadMergedRanges`, `ReadHyperlinks` | | Templates | `ExcelPackage.OpenTemplate(template, output)`, `Worksheets.Open(name)`, `Worksheet.TemplateStyle(cell)`, `ReplacedTemplateRows` | | Editing | `ExcelPackage.Edit(path)` / `Edit(input, output)`, `Worksheets.Edit(name)`, `Cells.Existing`, `Cells.Dimension`, `ExcelCell.Clear()` | | `SheetStream.AspNetCore` | `ExcelResults.Excel(fileName, build)`, `Results.Extensions.Excel(...)`, `ExcelResult` | @@ -289,7 +321,7 @@ The package ships a Roslyn analyzer, so the IDE warns before you run: **SS0001** | Edit and re-save an existing workbook | ✅ | ✅ | ❌ | ✅ (small workbooks, in memory) | | Read workbooks | ✅ | ✅ | ✅ streaming | ✅ streaming | | Formula calculation engine | ✅ | ✅ | ❌ | ❌ (Excel calculates on open) | -| Pivot tables | ✅ | ✅ | ❌ | ❌ | +| Pivot tables | ✅ | ✅ | ❌ | ✅ (refreshed on open) | ### Benchmarks @@ -318,7 +350,7 @@ Full results, the wide-sheet scenario, methodology, machine details and how to r - **You need to edit large workbooks in place.** SheetStream edits small workbooks (`ExcelPackage.Edit`) by holding the edited sheet in memory; for big files it fills templates or streams. Use ClosedXML (MIT) or EPPlus when you must rewrite a large arbitrary workbook. - **You need formulas evaluated in .NET**, for example to read a calculated total without Excel. Use ClosedXML or EPPlus, which have calculation engines. -- **You need pivot tables, sparklines, macros or encryption.** These are not supported. +- **You need sparklines, macros, encryption, or pivot tables with calculated fields, grouping or slicers.** These are not supported. - **You only map simple objects to and from sheets** and want the smallest API. MiniExcel is excellent at that. SheetStream is the right fit when you write large or styled exports, fill branded templates, stream reports from ASP.NET Core, or read big imports with flat memory, under a licence you never have to think about. diff --git a/docs/docfx.json b/docs/docfx.json index bb5cf77..64dad6c 100644 --- a/docs/docfx.json +++ b/docs/docfx.json @@ -16,7 +16,7 @@ { "files": [ "api/**.yml", "api/index.md" ] }, { "files": [ "*.md", "toc.yml" ], "exclude": [ "_site/**" ] } ], - "resource": [ { "files": [ "images/**" ] } ], + "resource": [ { "files": [ "images/**", "public/**" ] } ], "dest": "_site", "template": [ "default", "modern" ], "globalMetadata": { diff --git a/docs/images/logo.png b/docs/images/logo.png index 46a8549..1f2cc09 100644 Binary files a/docs/images/logo.png and b/docs/images/logo.png differ diff --git a/docs/index.md b/docs/index.md index f9bbede..945ad55 100644 --- a/docs/index.md +++ b/docs/index.md @@ -28,6 +28,10 @@ chart.AddSeries(values: "C2:C13", categories: "A2:A13", name: "=C1"); package.Save(); ``` +## Formulas and cached results + +`row.WriteFormula("SUM(C2:C101)")` writes the formula and asks Excel to calculate on open. Viewers that do not calculate (file previews, Quick Look, many mobile and web viewers, `ExcelReader`) show such a cell blank unless the file also carries the result. Turn on `sheet.CacheFormulaResults()` before the first row and SheetStream computes the result itself when the formula is simple enough: numbers, references to cells already written on the same sheet, ranges, `+ - * /`, `%`, parentheses and `SUM`, `AVERAGE`, `COUNT`, `COUNTA`, `MIN`, `MAX`, `ROUND`, `ABS`. Anything else (other functions, other sheets, rows not yet written, a division by zero) is written exactly as before, without a value; a wrong value is never written. The sheet keeps about 8 bytes per numeric cell to do this, which is why it is opt-in. When you already know the result, `WriteFormula(formula, cachedValue)` stores it directly. + ## Where to go next - [Migrating from EPPlus](migrating-from-epplus.md): a real exporter converted line by line. diff --git a/docs/migrating-from-epplus.md b/docs/migrating-from-epplus.md index c25f70d..59a4a57 100644 --- a/docs/migrating-from-epplus.md +++ b/docs/migrating-from-epplus.md @@ -162,4 +162,18 @@ The file is built into a temporary file and streamed to the response, so memory ## What does not translate -These EPPlus features have no SheetStream equivalent today: formula calculation (`Calculate()`), pivot tables, sparklines, editing an existing workbook in place (other than filling a template), VBA, encryption and digital signatures. If an exporter depends on one of them, keep that exporter on EPPlus 4.5.3.3 (LGPL) or another library. +These EPPlus features have no SheetStream equivalent today: formula calculation (`Calculate()`; SheetStream only writes cached results for simple aggregates, see `CacheFormulaResults`), pivot tables with calculated fields, grouping or slicers (plain pivots are supported and refreshed on open), sparklines, editing large arbitrary workbooks in place (small workbooks can be edited with `ExcelPackage.Edit`; large ones are filled as templates), VBA, encryption and digital signatures. If an exporter depends on one of them, keep that exporter on EPPlus 4.5.3.3 (LGPL) or another library. + +## After removing the EPPlus package + +Removing EPPlus can break code that never called it. EPPlus 4.5.3.3 depends on `System.Text.Encoding.CodePages` and registers the code-page encoding provider when it loads. Other libraries in the same application may rely on that registration without declaring it; **ExcelDataReader** is the usual case, because legacy `.xls` files need code-page encodings. With EPPlus gone, those imports fail, sometimes silently with empty text. + +Register the provider yourself at startup, before the first import: + +```csharp +// Program.cs, before building the app. On .NET Framework or netstandard targets add the +// System.Text.Encoding.CodePages package; on .NET 8 it is part of the shared framework. +System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance); +``` + +Then run one `.xls` import in a test to prove it. This was found while migrating a warehouse system whose exports moved to SheetStream but whose imports stayed on ExcelDataReader. For `.xlsx` imports, `ExcelReader` needs no provider. diff --git a/docs/pivot-tables.md b/docs/pivot-tables.md new file mode 100644 index 0000000..bd63131 --- /dev/null +++ b/docs/pivot-tables.md @@ -0,0 +1,74 @@ +# Pivot tables + +A pivot table summarises a range, or an Excel table, written earlier in the same workbook. SheetStream writes the pivot's layout and asks Excel to compute it when the file opens, so the result is a real, interactive pivot: users can drag fields, filter, drill and refresh exactly as if they had built it themselves. + +```csharp +using SheetStream; + +using var package = ExcelPackage.Create("sales.xlsx"); +var wb = package.Workbook; + +var data = wb.Worksheets.Add("Data"); +var loaded = data.LoadFromCollection(orders, new LoadOptions { TableName = "Orders" }); + +var report = wb.Worksheets.Add("Pivot"); +var pivot = report.AddPivotTable("SalesByRegion", loaded.Table!, row: 3, column: 1); +pivot.AddRowField("Region"); +pivot.AddColumnField("Category"); +pivot.AddDataField("Amount", PivotFunction.Sum, numberFormat: "$#,##0.00"); +pivot.AddDataField("OrderId", PivotFunction.Count, displayName: "Orders"); + +package.Save(); +``` + +## Two ways to name the source + +| Overload | Source | Field names come from | +|---|---|---| +| `AddPivotTable(name, excelTable, row, column)` | An `ExcelTable` from `AddTable` or `LoadOptions.TableName` (`LoadResult.Table`) | The table's columns. Excel re-reads the whole table on each refresh, so the pivot follows the table when rows are added later in Excel. | +| `AddPivotTable(name, sourceSheetName, sourceRange, row, column, headers)` | Any A1 range including its header row, e.g. `"A1:F500"` | The `headers` you pass. `LoadResult.Headers` and `LoadResult.Range` give you both for data written with a `LoadFrom*` call. | + +The header texts are needed because the source sheet has already been streamed into the file: SheetStream cannot read it back. Field names are matched case-insensitively; an unknown name throws an `ArgumentException` that lists the available headers. + +The source sheet may be the pivot's own sheet (a pivot next to its data) or any sheet added earlier. It must exist in the workbook when `AddPivotTable` is called. + +## Placing fields + +| Method | Excel area | Notes | +|---|---|---| +| `AddRowField(header)` | Rows | Several row fields nest in the order they are added. | +| `AddColumnField(header)` | Columns | | +| `AddFilterField(header)` | Filters (report filter / page field) | Each filter takes one row above the pivot body, followed by one blank row, as Excel lays it out. `row` is the first filter cell. | +| `AddDataField(header, function, displayName, numberFormat)` | Values | `PivotFunction`: Sum (default), Count, Average, Max, Min, Product, CountNumbers, StdDev, StdDevP, Var, VarP. The default caption is `"Sum of Amount"` style. With more than one data field Excel shows them side by side under a "Values" header. | + +A data field's caption must differ from every source header. Excel itself refuses `displayName: "Amount"` when a column is called Amount ("PivotTable field name already exists"), so SheetStream rejects it up front. + +Options on `PivotTable`: + +- `StyleName`: a built-in pivot style, default `"PivotStyleMedium9"`; also `ShowRowStripes` / `ShowColumnStripes`. +- `ShowRowGrandTotals` (the Grand Total column at the right) and `ShowColumnGrandTotals` (the Grand Total row at the bottom), both default `true`. +- `Compact`: `true` (default) stacks all row fields in one indented column, as Excel creates pivots; `false` gives outline form with one column per row field. + +Several pivots can share a sheet, sit on different sheets or point at the same source; each gets its own cache. Pivot names must be unique within the workbook. + +## How it works: refresh on open + +Streaming means the data is gone by the time the pivot is written, so SheetStream does not build the pivot cache (the list of distinct items per field and the record table) that Excel normally stores next to a pivot. Instead the file carries: + +- `xl/pivotCache/pivotCacheDefinitionN.xml` with the source reference, one `cacheField` per header, no items, `recordCount="0"`, `saveData="0"` and `refreshOnLoad="1"`; +- `xl/pivotCache/pivotCacheRecordsN.xml` with zero records (Excel expects the part to exist even when it is empty); +- `xl/pivotTables/pivotTableN.xml` with the fields, their areas, the style and a `location` sized for one item per axis. + +`refreshOnLoad` makes Excel read the source cells and compute every total the moment the workbook opens, exactly as if the user had clicked Refresh All, and it then lays the pivot out for the real number of items. Opening and saving the file in Excel stores a full cache like any other pivot. + +Version stamps are written as Excel 2016 writes them (`createdVersion`/`refreshedVersion` 6, `minRefreshableVersion` 3): Excel treats a cache whose refreshed version is older than the pivot's as stale and some builds then refuse to refresh it. + +`tests/excel-check-pivot.ps1` opens a workbook through Excel COM automation, checks that no part was dropped by a repair, calls `RefreshTable()` on every pivot and prints the refreshed cells; it is how the shipped pivots were verified. + +## Limits + +- **No cached data.** A viewer that does not run the pivot engine (web previews, some mobile apps, older LibreOffice builds) shows the header cells only, until the file has been opened and saved in Excel. If the report must render everywhere, write the summary as plain cells (`GroupBy` plus `WriteCell`) or formulas (`SUMIFS` on the table) instead. +- Supported layouts: row, column and filter fields, data fields with the eleven standard functions, number formats, grand totals, compact or outline form, built-in styles. +- Not supported: calculated fields and items, grouping (dates, numbers), slicers and timelines, custom sort orders, "show values as" (percent of total, running total), conditional formats inside the pivot, OLAP sources, external connections. +- Pivots cannot be added to template or edited workbooks (`OpenTemplate`, `Edit`), and existing pivots are not read or modified. +- A pivot whose source range has only a header row is refused (`ArgumentException`), as Excel cannot build a cache from it. diff --git a/docs/public/main.css b/docs/public/main.css new file mode 100644 index 0000000..4428faa --- /dev/null +++ b/docs/public/main.css @@ -0,0 +1,9 @@ +/* Site-specific tweaks for the DocFX "modern" template (picked up from public/main.css). */ + +/* The template shows the logo at the image's own size; keep it a navbar-sized icon, scaled + proportionally. The P4 Software logo must never be stretched. */ +#logo, +.navbar-brand img { + height: 32px; + width: auto; +} diff --git a/docs/toc.yml b/docs/toc.yml index 33c523b..454e7be 100644 --- a/docs/toc.yml +++ b/docs/toc.yml @@ -6,6 +6,8 @@ href: templates.md - name: Editing existing workbooks href: editing-workbooks.md +- name: Pivot tables + href: pivot-tables.md - name: Trimming, Native AOT and the source generator href: aot-and-source-generator.md - name: Analyzers diff --git a/samples/SheetStream.Sample/Showcase.cs b/samples/SheetStream.Sample/Showcase.cs index 19ec81e..1bfa2a8 100644 --- a/samples/SheetStream.Sample/Showcase.cs +++ b/samples/SheetStream.Sample/Showcase.cs @@ -128,7 +128,17 @@ public static void Run(string path) if (File.Exists(banner)) summary.AddPicture(banner, 20, 4); summary.Protect(configure: o => o.AllowSelectLockedCells = true); - // ---- Sheet 3: outline grouping and a hidden lookup sheet. + // ---- Sheet 3: a pivot table over the invoice table. The cache is empty; Excel fills it on open. + var pivotSheet = wb.Worksheets.Add("Pivot"); + pivotSheet.Column(1).SetWidth(22); + pivotSheet.Row(1, title).WriteCell("Amount by customer and status"); + var pivot = pivotSheet.AddPivotTable("InvoicePivot", loaded.Table!, row: 3, column: 1); + pivot.AddRowField("Customer"); + pivot.AddColumnField("Status"); + pivot.AddDataField("Amount", PivotFunction.Sum, numberFormat: "$#,##0.00"); + pivot.AddDataField("Invoice #", PivotFunction.Count, "Invoices"); + + // ---- Sheet 4: outline grouping and a hidden lookup sheet. var grouped = wb.Worksheets.Add("Grouped"); grouped.Row().WriteCell("Region", header).WriteCell("Sales", header); foreach (var region in new[] { "North", "South" }) diff --git a/src/SheetStream/ExcelPackage.cs b/src/SheetStream/ExcelPackage.cs index cde1205..53b2af3 100644 --- a/src/SheetStream/ExcelPackage.cs +++ b/src/SheetStream/ExcelPackage.cs @@ -327,11 +327,29 @@ private void WriteWorkbookPart(XmlWriter w) w.WriteEndElement(); } + // CT_Workbook continues: oleSize, customWorkbookViews, pivotCaches. Each cache is a workbook-level + // relationship; the cacheId here is what the pivot table definitions point at. + if (Workbook.PivotCaches.Count > 0) + { + w.WriteStartElement("pivotCaches"); + for (var i = 0; i < Workbook.PivotCaches.Count; i++) + { + w.WriteStartElement("pivotCache"); + w.WriteAttributeString("cacheId", Workbook.PivotCaches[i].ToString(CultureInfo.InvariantCulture)); + w.WriteAttributeString("id", OpenXmlNamespaces.Relationships, PivotCacheRelationshipId(i)); + w.WriteEndElement(); + } + w.WriteEndElement(); + } + w.WriteEndElement(); w.WriteEndDocument(); } - /// xl/_rels/workbook.xml.rels: one relationship per sheet, then styles and shared strings. + /// Relationship id of the i-th pivot cache: allocated after the sheets, styles and shared strings. + private string PivotCacheRelationshipId(int index) => "rId" + (Workbook.Worksheets.Count + 3 + index).ToString(CultureInfo.InvariantCulture); + + /// xl/_rels/workbook.xml.rels: one relationship per sheet, then styles, shared strings and pivot caches. private void WriteWorkbookRelationships(XmlWriter w) { w.WriteStartDocument(true); @@ -341,6 +359,8 @@ private void WriteWorkbookRelationships(XmlWriter w) var next = Workbook.Worksheets.Count + 1; WriteRelationship(w, "rId" + next.ToString(CultureInfo.InvariantCulture), OpenXmlNamespaces.RelStyles, "styles.xml"); WriteRelationship(w, "rId" + (next + 1).ToString(CultureInfo.InvariantCulture), OpenXmlNamespaces.RelSharedStrings, "sharedStrings.xml"); + for (var i = 0; i < Workbook.PivotCaches.Count; i++) + WriteRelationship(w, PivotCacheRelationshipId(i), OpenXmlNamespaces.RelPivotCacheDefinition, "pivotCache/pivotCacheDefinition" + Workbook.PivotCaches[i].ToString(CultureInfo.InvariantCulture) + ".xml"); w.WriteEndElement(); w.WriteEndDocument(); } diff --git a/src/SheetStream/Features/ExcelTable.cs b/src/SheetStream/Features/ExcelTable.cs index 12981ef..3ad9f32 100644 --- a/src/SheetStream/Features/ExcelTable.cs +++ b/src/SheetStream/Features/ExcelTable.cs @@ -72,6 +72,24 @@ internal bool NeedsHeaderCapture } } + /// + /// The column names once every header is known (given explicitly or captured from the header row), or + /// null while some are still missing. Pivot tables read their source field names from here. + /// + internal IReadOnlyList? ColumnNames + { + get + { + var names = new string[_columnNames.Length]; + for (var i = 0; i < names.Length; i++) + { + if (_columnNames[i] == null) return null; + names[i] = _columnNames[i]!; + } + return names; + } + } + /// Records the text written into a header cell of this table. internal void CaptureHeader(int column, string text) { diff --git a/src/SheetStream/Features/LoadOptions.cs b/src/SheetStream/Features/LoadOptions.cs index 4a37e8b..ce1168c 100644 --- a/src/SheetStream/Features/LoadOptions.cs +++ b/src/SheetStream/Features/LoadOptions.cs @@ -69,15 +69,22 @@ public SheetColumnAttribute(string name) /// The cells a load call wrote, for follow-up formatting such as charts or totals. public readonly struct LoadResult { - internal LoadResult(int firstRow, int lastRow, int firstColumn, int lastColumn, int dataRows) + private readonly string[]? _headers; + + internal LoadResult(int firstRow, int lastRow, int firstColumn, int lastColumn, int dataRows, string[]? headers = null, ExcelTable? table = null) { FirstRow = firstRow; LastRow = lastRow; FirstColumn = firstColumn; LastColumn = lastColumn; DataRows = dataRows; + _headers = headers; + Table = table; } + /// The Excel table created over the data when was set, otherwise null. + public ExcelTable? Table { get; } + /// First row written (the header row when headers are printed). public int FirstRow { get; } @@ -92,5 +99,17 @@ internal LoadResult(int firstRow, int lastRow, int firstColumn, int lastColumn, /// Number of data rows (excluding the header). public int DataRows { get; } + + /// + /// The column header texts in column order (empty when nothing was loaded). Pass them to + /// on a later + /// sheet: the source rows have already been streamed out, so the pivot cannot read them back. + /// + public IReadOnlyList Headers => _headers ?? Array.Empty(); + + /// The loaded block as an A1 range including the header row, e.g. "A2:F62"; the pivot source range. + public string Range => LastColumn < FirstColumn + ? Internal.CellReference.ToA1(FirstRow, FirstColumn) + : Internal.CellReference.ToRange(FirstRow, FirstColumn, LastRow < FirstRow ? FirstRow : LastRow, LastColumn); } } diff --git a/src/SheetStream/Features/PivotTable.cs b/src/SheetStream/Features/PivotTable.cs new file mode 100644 index 0000000..a8fea2e --- /dev/null +++ b/src/SheetStream/Features/PivotTable.cs @@ -0,0 +1,550 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Xml; +using SheetStream.Internal; + +namespace SheetStream +{ + /// Aggregation applied by a pivot data field (). + public enum PivotFunction + { + /// Sum of the values (Excel's default for numeric columns). + Sum, + /// Number of non-empty values. + Count, + /// Arithmetic mean. + Average, + /// Largest value. + Max, + /// Smallest value. + Min, + /// Product of the values. + Product, + /// Number of numeric values. + CountNumbers, + /// Sample standard deviation. + StdDev, + /// Population standard deviation. + StdDevP, + /// Sample variance. + Var, + /// Population variance. + VarP, + } + + /// + /// A pivot table summarising a range (or an Excel table) written earlier in the same workbook. Create it with + /// or + /// , then place source columns on the + /// row, column, filter and data areas by their header text. + /// + /// + /// + /// SheetStream streams every sheet forward-only, so by the time the pivot is written the source data has + /// already left memory. The file therefore carries the pivot's layout and an empty cache flagged + /// refreshOnLoad: Excel reads the source cells and computes the totals the moment the workbook opens, + /// exactly as if the user had pressed Refresh. Viewers that do not run the pivot engine (some web previews, + /// LibreOffice in certain versions) show an empty pivot until the file has been opened and saved in Excel. + /// + /// + /// Not supported: calculated fields, grouping, slicers, custom item sorting, reading or editing pivots of an + /// existing workbook, and pivots on template or edited workbooks. + /// + /// + /// + /// + /// var pivot = summary.AddPivotTable("ByCustomer", invoicesTable, row: 3, column: 1); + /// pivot.AddRowField("Customer"); + /// pivot.AddColumnField("Status"); + /// pivot.AddDataField("Amount", PivotFunction.Sum, numberFormat: "$#,##0.00"); + /// pivot.AddDataField("Invoice #", PivotFunction.Count, "Invoices"); + /// + /// + public sealed class PivotTable + { + private readonly string[] _headers; + private readonly List _rowFields = new List(); + private readonly List _columnFields = new List(); + private readonly List _pageFields = new List(); + private readonly List _dataFields = new List(); + + /// One entry of the Values area. + private readonly struct DataField + { + public DataField(int field, PivotFunction function, string name, string? numberFormat) + { + Field = field; + Function = function; + Name = name; + NumberFormat = numberFormat; + } + + public int Field { get; } + public PivotFunction Function { get; } + public string Name { get; } + public string? NumberFormat { get; } + } + + internal PivotTable(Worksheet sheet, int number, string name, string sourceSheet, RangeReference sourceRange, string? sourceTableName, IReadOnlyList headers, int row, int column) + { + Sheet = sheet; + Number = number; + Name = name; + SourceSheetName = sourceSheet; + SourceRange = sourceRange; + SourceTableName = sourceTableName; + Row = row; + Column = column; + _headers = new string[headers.Count]; + var seen = new HashSet(StringComparer.OrdinalIgnoreCase); + for (var i = 0; i < _headers.Length; i++) + { + var h = headers[i]; + if (string.IsNullOrWhiteSpace(h)) + throw new ArgumentException($"Header {i + 1} of pivot table '{name}' is empty. Every source column needs a header text; Excel uses it as the field name.", nameof(headers)); + // Excel refuses a cache with two fields of the same name ("The PivotTable field name is not valid"). + if (!seen.Add(h)) + throw new ArgumentException($"Pivot table '{name}' has two source columns named '{h}'. Excel requires unique header texts.", nameof(headers)); + _headers[i] = h; + } + } + + internal Worksheet Sheet { get; } + + /// Workbook-wide pivot number: the part number of the pivot table, its cache definition and its records. + internal int Number { get; } + + internal RangeReference SourceRange { get; } + + /// Table name when the source is an Excel table; the cache then follows the table as it grows. + internal string? SourceTableName { get; } + + /// Pivot table name shown in Excel's PivotTable Analyze ribbon. Unique within the workbook. + public string Name { get; } + + /// Name of the sheet holding the source data. + public string SourceSheetName { get; } + + /// 1-based row of the pivot's top-left cell (the first filter cell when there are filter fields). + public int Row { get; } + + /// 1-based column of the pivot's top-left cell. + public int Column { get; } + + /// The source header texts, in column order; these are the names Add*Field accepts. + public IReadOnlyList Headers => _headers; + + /// Built-in pivot style, e.g. "PivotStyleMedium9" (default), "PivotStyleLight16", "PivotStyleDark2". + public string StyleName { get; set; } = "PivotStyleMedium9"; + + /// Show the Grand Total column at the right (the total of each row). Default true. + public bool ShowRowGrandTotals { get; set; } = true; + + /// Show the Grand Total row at the bottom (the total of each column). Default true. + public bool ShowColumnGrandTotals { get; set; } = true; + + /// + /// Compact form (default): all row fields share one indented column, as Excel creates pivots. Set to + /// false for outline form, where each row field gets its own column. + /// + public bool Compact { get; set; } = true; + + /// Shade alternate rows with the style's banding. + public bool ShowRowStripes { get; set; } + + /// Shade alternate columns with the style's banding. + public bool ShowColumnStripes { get; set; } + + /// Puts a source column on the row axis. Fields nest in the order they are added. + /// Header text of the source column, as passed in headers or captured from the table. + public PivotTable AddRowField(string sourceColumnHeader) + { + Sheet.EnsureWritable(); + _rowFields.Add(Resolve(sourceColumnHeader, nameof(sourceColumnHeader))); + return this; + } + + /// Puts a source column on the column axis. + public PivotTable AddColumnField(string sourceColumnHeader) + { + Sheet.EnsureWritable(); + _columnFields.Add(Resolve(sourceColumnHeader, nameof(sourceColumnHeader))); + return this; + } + + /// + /// Adds a report filter (page field) above the pivot. Each filter takes one row; the pivot body starts + /// below them with one blank row in between, as Excel lays it out. + /// + public PivotTable AddFilterField(string sourceColumnHeader) + { + Sheet.EnsureWritable(); + _pageFields.Add(Resolve(sourceColumnHeader, nameof(sourceColumnHeader))); + return this; + } + + /// + /// Adds a value to the data area. With more than one data field Excel shows them side by side under a + /// "Values" column header. + /// + /// Header text of the source column to aggregate. + /// Aggregation. Default sum. + /// + /// Caption shown in the pivot, default "Sum of Amount" style. Must differ from every source header: + /// Excel reports "PivotTable field name already exists" otherwise, so that is rejected here. + /// + /// Excel number format for the values, e.g. "#,##0.00", or null for General. + public PivotTable AddDataField(string sourceColumnHeader, PivotFunction function = PivotFunction.Sum, string? displayName = null, string? numberFormat = null) + { + Sheet.EnsureWritable(); + var field = Resolve(sourceColumnHeader, nameof(sourceColumnHeader)); + var name = displayName ?? Caption(function) + " of " + _headers[field]; + if (string.IsNullOrWhiteSpace(name)) throw new ArgumentException("Data field display name cannot be empty.", nameof(displayName)); + foreach (var h in _headers) + if (string.Equals(h, name, StringComparison.OrdinalIgnoreCase)) + throw new ArgumentException($"Data field name '{name}' equals the source column '{h}'. Excel refuses a data field named like a source field; pick another displayName (for example '{name} total').", nameof(displayName)); + foreach (var d in _dataFields) + if (string.Equals(d.Name, name, StringComparison.OrdinalIgnoreCase)) + throw new ArgumentException($"Pivot table '{Name}' already has a data field named '{name}'. Give the second one a displayName.", nameof(displayName)); + _dataFields.Add(new DataField(field, function, name, string.IsNullOrEmpty(numberFormat) ? null : numberFormat)); + return this; + } + + /// Maps a header text to its 0-based field index. Exact match first, then case-insensitive. + private int Resolve(string header, string parameterName) + { + if (header == null) throw new ArgumentNullException(parameterName); + for (var i = 0; i < _headers.Length; i++) if (_headers[i] == header) return i; + for (var i = 0; i < _headers.Length; i++) if (string.Equals(_headers[i], header, StringComparison.OrdinalIgnoreCase)) return i; + throw new ArgumentException($"Pivot table '{Name}' has no source column '{header}'. Available headers: {string.Join(", ", _headers)}.", parameterName); + } + + private static string Caption(PivotFunction function) + { + switch (function) + { + case PivotFunction.Sum: return "Sum"; + case PivotFunction.Count: return "Count"; + case PivotFunction.Average: return "Average"; + case PivotFunction.Max: return "Max"; + case PivotFunction.Min: return "Min"; + case PivotFunction.Product: return "Product"; + case PivotFunction.CountNumbers: return "Count"; + case PivotFunction.StdDev: return "StdDev"; + case PivotFunction.StdDevP: return "StdDevp"; + case PivotFunction.Var: return "Var"; + case PivotFunction.VarP: return "Varp"; + default: throw new ArgumentOutOfRangeException(nameof(function)); + } + } + + /// The ST_DataConsolidateFunction token of a function. + private static string SubtotalToken(PivotFunction function) + { + switch (function) + { + case PivotFunction.Sum: return "sum"; + case PivotFunction.Count: return "count"; + case PivotFunction.Average: return "average"; + case PivotFunction.Max: return "max"; + case PivotFunction.Min: return "min"; + case PivotFunction.Product: return "product"; + case PivotFunction.CountNumbers: return "countNums"; + case PivotFunction.StdDev: return "stdDev"; + case PivotFunction.StdDevP: return "stdDevp"; + case PivotFunction.Var: return "var"; + case PivotFunction.VarP: return "varp"; + default: throw new ArgumentOutOfRangeException(nameof(function)); + } + } + + /// + /// The cells the pivot body occupies before Excel refreshes it, as Excel's location element + /// describes it: the body (without the filter rows), the row offset of the header rows and the first data + /// row, and the column offset of the first data column. The source data is unknown here, so the body is + /// sized for one item per axis; Excel resizes it on refresh. Internal so tests can pin the arithmetic. + /// + internal (RangeReference Ref, int FirstHeaderRow, int FirstDataRow, int FirstDataCol) ComputeLocation() + { + var rowCount = _rowFields.Count; + var colCount = _columnFields.Count; + var dataCount = _dataFields.Count; + var valueColumns = Math.Max(1, dataCount); + + // Filters sit above the body, one per row, followed by one blank row (Excel's layout). + var bodyRow = Row + (_pageFields.Count > 0 ? _pageFields.Count + 1 : 0); + + // Header rows above the first data row. Without column fields Excel uses a single header row + // ("Row Labels | Sum of X [| Count of Y]"). Column fields add a caption row ("Sum of X | Column Labels"), + // and several data fields under column fields add a "Values" row as well. + var firstDataRow = colCount == 0 ? 1 : dataCount > 1 ? 3 : 2; + + // Compact form stacks every row field in one column; outline form gives each its own. + var firstDataCol = rowCount == 0 ? 0 : Compact ? 1 : rowCount; + + var rows = firstDataRow + 1 + (rowCount > 0 && ShowColumnGrandTotals ? 1 : 0); + var cols = firstDataCol + valueColumns + (colCount > 0 && ShowRowGrandTotals ? valueColumns : 0); + + var lastRow = Math.Min(CellReference.MaxRows, bodyRow + rows - 1); + var lastCol = Math.Min(CellReference.MaxColumns, Column + cols - 1); + return (new RangeReference(bodyRow, Column, lastRow, lastCol), 1, firstDataRow, firstDataCol); + } + + /// Throws when the pivot cannot be written; called before any part is opened so a failure leaves the archive consistent. + internal void Validate() + { + if (_rowFields.Count + _columnFields.Count + _pageFields.Count + _dataFields.Count == 0) + throw new InvalidOperationException($"Pivot table '{Name}' has no fields. Call AddRowField / AddColumnField / AddFilterField / AddDataField before the sheet is sealed."); + ComputeLocation(); // Throws when the filter rows push the body past the last row of the sheet. + } + + /// + /// Writes xl/pivotCache/pivotCacheDefinitionN.xml. The cache is empty and flagged + /// refreshOnLoad; saveData="0" tells Excel the records part carries nothing worth keeping. + /// + internal void WriteCacheDefinition(XmlWriter w, string recordsRelId) + { + w.WriteStartDocument(true); + w.WriteStartElement("pivotCacheDefinition", OpenXmlNamespaces.Main); + w.WriteAttributeString("xmlns", "r", null, OpenXmlNamespaces.Relationships); + w.WriteAttributeString("id", OpenXmlNamespaces.Relationships, recordsRelId); + w.WriteAttributeString("saveData", "0"); + w.WriteAttributeString("refreshOnLoad", "1"); + w.WriteAttributeString("refreshedBy", "SheetStream"); + // Version stamps as Excel 2016 writes them. A refreshedVersion below the pivot's updatedVersion + // makes Excel treat the cache as stale and some builds then refuse to refresh it. + w.WriteAttributeString("createdVersion", "6"); + w.WriteAttributeString("refreshedVersion", "6"); + w.WriteAttributeString("minRefreshableVersion", "3"); + w.WriteAttributeString("recordCount", "0"); + + // CT_PivotCacheDefinition order: cacheSource, cacheFields. + w.WriteStartElement("cacheSource"); + w.WriteAttributeString("type", "worksheet"); + w.WriteStartElement("worksheetSource"); + if (SourceTableName != null) + { + // A table source keeps the pivot in step with the table's current size on every refresh. + w.WriteAttributeString("name", SourceTableName); + } + else + { + w.WriteAttributeString("ref", SourceRange.ToString()); + w.WriteAttributeString("sheet", SourceSheetName); + } + w.WriteEndElement(); + w.WriteEndElement(); + + w.WriteStartElement("cacheFields"); + w.WriteAttributeString("count", _headers.Length.ToString(CultureInfo.InvariantCulture)); + foreach (var header in _headers) + { + w.WriteStartElement("cacheField"); + w.WriteAttributeString("name", XmlTextUtil.Sanitize(header)); + w.WriteAttributeString("numFmtId", "0"); + // No items: Excel rebuilds the shared item list from the source on load. + w.WriteStartElement("sharedItems"); + w.WriteEndElement(); + w.WriteEndElement(); + } + w.WriteEndElement(); + + w.WriteEndElement(); + w.WriteEndDocument(); + } + + /// + /// Writes xl/pivotCache/pivotCacheRecordsN.xml with zero records. Excel expects the part to exist + /// even when the definition says saveData="0"; without it some versions repair the file and drop the pivot. + /// + internal static void WriteCacheRecords(XmlWriter w) + { + w.WriteStartDocument(true); + w.WriteStartElement("pivotCacheRecords", OpenXmlNamespaces.Main); + w.WriteAttributeString("xmlns", "r", null, OpenXmlNamespaces.Relationships); + w.WriteAttributeString("count", "0"); + w.WriteEndElement(); + w.WriteEndDocument(); + } + + /// + /// Writes xl/pivotTables/pivotTableN.xml. CT_pivotTableDefinition order: location, pivotFields, + /// rowFields, colFields, pageFields, dataFields, pivotTableStyleInfo (rowItems/colItems are omitted; Excel + /// derives them when it refreshes the empty cache). + /// + internal void WriteDefinition(XmlWriter w, int cacheId) + { + var (location, firstHeaderRow, firstDataRow, firstDataCol) = ComputeLocation(); + + w.WriteStartDocument(true); + w.WriteStartElement("pivotTableDefinition", OpenXmlNamespaces.Main); + w.WriteAttributeString("name", XmlTextUtil.Sanitize(Name)); + w.WriteAttributeString("cacheId", cacheId.ToString(CultureInfo.InvariantCulture)); + w.WriteAttributeString("applyNumberFormats", "0"); + w.WriteAttributeString("applyBorderFormats", "0"); + w.WriteAttributeString("applyFontFormats", "0"); + w.WriteAttributeString("applyPatternFormats", "0"); + w.WriteAttributeString("applyAlignmentFormats", "0"); + w.WriteAttributeString("applyWidthHeightFormats", "1"); + w.WriteAttributeString("dataCaption", "Values"); + w.WriteAttributeString("updatedVersion", "6"); + w.WriteAttributeString("minRefreshableVersion", "3"); + w.WriteAttributeString("useAutoFormatting", "1"); + if (!ShowRowGrandTotals) w.WriteAttributeString("rowGrandTotals", "0"); + if (!ShowColumnGrandTotals) w.WriteAttributeString("colGrandTotals", "0"); + w.WriteAttributeString("itemPrintTitles", "1"); + w.WriteAttributeString("createdVersion", "6"); + w.WriteAttributeString("indent", "0"); + if (!Compact) w.WriteAttributeString("compact", "0"); + w.WriteAttributeString("outline", "1"); + w.WriteAttributeString("outlineData", "1"); + if (!Compact) w.WriteAttributeString("compactData", "0"); + w.WriteAttributeString("multipleFieldFilters", "0"); + + w.WriteStartElement("location"); + w.WriteAttributeString("ref", location.ToString()); + w.WriteAttributeString("firstHeaderRow", firstHeaderRow.ToString(CultureInfo.InvariantCulture)); + w.WriteAttributeString("firstDataRow", firstDataRow.ToString(CultureInfo.InvariantCulture)); + w.WriteAttributeString("firstDataCol", firstDataCol.ToString(CultureInfo.InvariantCulture)); + if (_pageFields.Count > 0) + { + w.WriteAttributeString("rowPageCount", _pageFields.Count.ToString(CultureInfo.InvariantCulture)); + w.WriteAttributeString("colPageCount", "1"); + } + w.WriteEndElement(); + + // One pivotField per cache field, in cache order; the axis and dataField flags say where it is used. + w.WriteStartElement("pivotFields"); + w.WriteAttributeString("count", _headers.Length.ToString(CultureInfo.InvariantCulture)); + for (var i = 0; i < _headers.Length; i++) + { + var axis = _rowFields.Contains(i) ? "axisRow" : _columnFields.Contains(i) ? "axisCol" : _pageFields.Contains(i) ? "axisPage" : null; + var isData = false; + foreach (var d in _dataFields) isData |= d.Field == i; + + w.WriteStartElement("pivotField"); + if (axis != null) w.WriteAttributeString("axis", axis); + if (isData) w.WriteAttributeString("dataField", "1"); + if (!Compact) w.WriteAttributeString("compact", "0"); + w.WriteAttributeString("showAll", "0"); + if (axis != null) + { + // An axis field needs an item list; with an empty cache the only item is the "default" (subtotal) placeholder. + w.WriteStartElement("items"); + w.WriteAttributeString("count", "1"); + w.WriteStartElement("item"); + w.WriteAttributeString("t", "default"); + w.WriteEndElement(); + w.WriteEndElement(); + } + w.WriteEndElement(); + } + w.WriteEndElement(); + + WriteAxis(w, "rowFields", _rowFields, false); + // Several data fields go on the column axis as the pseudo field -2 ("Values"), Excel's default placement. + WriteAxis(w, "colFields", _columnFields, _dataFields.Count > 1); + + if (_pageFields.Count > 0) + { + w.WriteStartElement("pageFields"); + w.WriteAttributeString("count", _pageFields.Count.ToString(CultureInfo.InvariantCulture)); + foreach (var f in _pageFields) + { + w.WriteStartElement("pageField"); + w.WriteAttributeString("fld", f.ToString(CultureInfo.InvariantCulture)); + w.WriteAttributeString("hier", "-1"); // Not an OLAP hierarchy. + w.WriteEndElement(); + } + w.WriteEndElement(); + } + + if (_dataFields.Count > 0) + { + w.WriteStartElement("dataFields"); + w.WriteAttributeString("count", _dataFields.Count.ToString(CultureInfo.InvariantCulture)); + foreach (var d in _dataFields) + { + w.WriteStartElement("dataField"); + w.WriteAttributeString("name", XmlTextUtil.Sanitize(d.Name)); + w.WriteAttributeString("fld", d.Field.ToString(CultureInfo.InvariantCulture)); + w.WriteAttributeString("subtotal", SubtotalToken(d.Function)); + w.WriteAttributeString("baseField", "0"); + w.WriteAttributeString("baseItem", "0"); + if (d.NumberFormat != null) + w.WriteAttributeString("numFmtId", Sheet.Workbook.Styles.RegisterNumberFormat(d.NumberFormat).ToString(CultureInfo.InvariantCulture)); + w.WriteEndElement(); + } + w.WriteEndElement(); + } + + w.WriteStartElement("pivotTableStyleInfo"); + w.WriteAttributeString("name", StyleName); + w.WriteAttributeString("showRowHeaders", "1"); + w.WriteAttributeString("showColHeaders", "1"); + w.WriteAttributeString("showRowStripes", ShowRowStripes ? "1" : "0"); + w.WriteAttributeString("showColStripes", ShowColumnStripes ? "1" : "0"); + w.WriteAttributeString("showLastColumn", "1"); + w.WriteEndElement(); + + w.WriteEndElement(); + w.WriteEndDocument(); + } + + private static void WriteAxis(XmlWriter w, string element, List fields, bool appendValuesField) + { + var count = fields.Count + (appendValuesField ? 1 : 0); + if (count == 0) return; + w.WriteStartElement(element); + w.WriteAttributeString("count", count.ToString(CultureInfo.InvariantCulture)); + foreach (var f in fields) + { + w.WriteStartElement("field"); + w.WriteAttributeString("x", f.ToString(CultureInfo.InvariantCulture)); + w.WriteEndElement(); + } + if (appendValuesField) + { + w.WriteStartElement("field"); + w.WriteAttributeString("x", "-2"); + w.WriteEndElement(); + } + w.WriteEndElement(); + } + } + + /// + /// Writes the three parts of one pivot table and their relationships: the cache definition (related to + /// the records part), the empty records part, and the pivot table definition (related to the cache). The + /// sheet's own .rels entry and the workbook's pivotCaches entry are added by the callers. + /// + internal static class PivotTableWriter + { + public static void Write(ExcelPackage package, Workbook workbook, PivotTable pivot) + { + pivot.Validate(); + var n = pivot.Number.ToString(CultureInfo.InvariantCulture); + + var cacheRels = new RelationshipSet(); + var recordsRelId = cacheRels.Add(OpenXmlNamespaces.RelPivotCacheRecords, "pivotCacheRecords" + n + ".xml"); + var cachePart = "xl/pivotCache/pivotCacheDefinition" + n + ".xml"; + using (var w = package.OpenPart(cachePart)) pivot.WriteCacheDefinition(w, recordsRelId); + package.RegisterPart(cachePart, OpenXmlNamespaces.CtPivotCacheDefinition); + using (var w = package.OpenPart("xl/pivotCache/_rels/pivotCacheDefinition" + n + ".xml.rels")) cacheRels.WriteTo(w); + + var recordsPart = "xl/pivotCache/pivotCacheRecords" + n + ".xml"; + using (var w = package.OpenPart(recordsPart)) PivotTable.WriteCacheRecords(w); + package.RegisterPart(recordsPart, OpenXmlNamespaces.CtPivotCacheRecords); + + // The cache id is the part number; workbook.xml lists the same number against the cache relationship. + workbook.PivotCaches.Add(pivot.Number); + var tableRels = new RelationshipSet(); + tableRels.Add(OpenXmlNamespaces.RelPivotCacheDefinition, "../pivotCache/pivotCacheDefinition" + n + ".xml"); + var tablePart = "xl/pivotTables/pivotTable" + n + ".xml"; + using (var w = package.OpenPart(tablePart)) pivot.WriteDefinition(w, pivot.Number); + package.RegisterPart(tablePart, OpenXmlNamespaces.CtPivotTable); + using (var w = package.OpenPart("xl/pivotTables/_rels/pivotTable" + n + ".xml.rels")) tableRels.WriteTo(w); + } + } +} diff --git a/src/SheetStream/Internal/FormulaEvaluator.cs b/src/SheetStream/Internal/FormulaEvaluator.cs new file mode 100644 index 0000000..c19ed92 --- /dev/null +++ b/src/SheetStream/Internal/FormulaEvaluator.cs @@ -0,0 +1,437 @@ +using System; +using System.Globalization; + +namespace SheetStream.Internal +{ + /// + /// Evaluates the small subset of Excel formulas that business reports use for totals, so their result can + /// be written as the cell's cached value. This is deliberately not a formula engine: it knows numbers, + /// same-sheet cell and range references, + - * /, unary minus, a percent suffix, parentheses and + /// the functions SUM, AVERAGE, COUNT, COUNTA, MIN, MAX, ROUND and ABS. Everything else, and every + /// reference to a cell that has not been written yet, makes evaluation fail so the formula is written + /// without a cached value. Failing is always preferred over guessing: a viewer that shows a blank is + /// better than one that shows a wrong total. + /// + internal static class FormulaEvaluator + { + /// + /// Tries to compute (without the leading =) for the cell being + /// written at /. Only cells already written (rows above, + /// or cells to the left on the same row) may be referenced. + /// + /// False when the formula uses anything outside the supported subset, references a cell that is + /// not yet known, divides by zero or produces a non-finite number. + public static bool TryEvaluate(string formula, FormulaValueStore values, int row, int column, string sheetName, out double result) + { + var parser = new Parser(formula, values, row, column, sheetName); + result = parser.Run(); + if (parser.Failed || double.IsNaN(result) || double.IsInfinity(result)) + { + result = 0; + return false; + } + return true; + } + + private enum Function { Sum, Average, Count, CountA, Min, Max, Round, Abs } + + /// + /// Recursive-descent parser that evaluates as it parses. Unsupported input sets + /// instead of throwing, because a formula per row on a large export would otherwise pay for an + /// exception per row. + /// + private sealed class Parser + { + private readonly string _s; + private readonly FormulaValueStore _values; + private readonly int _row; + private readonly int _column; + private readonly string _sheetName; + private int _pos; + + public Parser(string formula, FormulaValueStore values, int row, int column, string sheetName) + { + _s = formula; + _values = values; + _row = row; + _column = column; + _sheetName = sheetName; + } + + public bool Failed { get; private set; } + + public double Run() + { + SkipSpaces(); + var v = ParseExpression(); + SkipSpaces(); + if (_pos != _s.Length) Fail(); + return v; + } + + private double Fail() + { + Failed = true; + return 0; + } + + // expression := term (('+' | '-') term)* + private double ParseExpression() + { + var left = ParseTerm(); + while (!Failed) + { + SkipSpaces(); + if (_pos >= _s.Length) break; + var op = _s[_pos]; + if (op != '+' && op != '-') break; + _pos++; + var right = ParseTerm(); + left = op == '+' ? left + right : left - right; + } + return left; + } + + // term := factor (('*' | '/') factor)* + private double ParseTerm() + { + var left = ParseFactor(); + while (!Failed) + { + SkipSpaces(); + if (_pos >= _s.Length) break; + var op = _s[_pos]; + if (op != '*' && op != '/') break; + _pos++; + var right = ParseFactor(); + if (op == '*') left *= right; + else if (right == 0) return Fail(); // Excel shows #DIV/0!; an error is not something to cache. + else left /= right; + } + return left; + } + + // factor := ('-' | '+') factor | primary '%'* + private double ParseFactor() + { + SkipSpaces(); + if (_pos < _s.Length && (_s[_pos] == '-' || _s[_pos] == '+')) + { + var negative = _s[_pos] == '-'; + _pos++; + var v = ParseFactor(); + return negative ? -v : v; + } + var value = ParsePrimary(); + while (!Failed) + { + SkipSpaces(); + if (_pos < _s.Length && _s[_pos] == '%') + { + _pos++; + value /= 100; + } + else break; + } + return value; + } + + // primary := number | '(' expression ')' | function '(' arguments ')' | cell reference + private double ParsePrimary() + { + SkipSpaces(); + if (_pos >= _s.Length) return Fail(); + var c = _s[_pos]; + if ((c >= '0' && c <= '9') || c == '.') return ParseNumber(); + if (c == '(') + { + _pos++; + var v = ParseExpression(); + SkipSpaces(); + if (_pos >= _s.Length || _s[_pos] != ')') return Fail(); + _pos++; + return v; + } + + // A reference (possibly sheet-qualified), or a function name followed by '('. + if (c == '\'' || c == '$' || IsLetter(c) || c == '_') + { + var start = _pos; + if (c != '\'' && c != '$') + { + var name = ReadIdentifier(); + SkipSpaces(); + if (_pos < _s.Length && _s[_pos] == '(') + { + _pos++; + return ParseFunctionCall(name); + } + _pos = start; + } + if (!TryParseReference(out var range, out var isRange)) return Fail(); + if (isRange) return Fail(); // A range only makes sense inside an aggregate function. + return CellValueForArithmetic(range.FromRow, range.FromColumn); + } + return Fail(); // Strings, comparisons, names, '&', '^' and anything else. + } + + private double ParseNumber() + { + var start = _pos; + while (_pos < _s.Length && ((_s[_pos] >= '0' && _s[_pos] <= '9') || _s[_pos] == '.')) _pos++; + if (_pos < _s.Length && (_s[_pos] == 'e' || _s[_pos] == 'E')) + { + // Only an exponent when digits follow, otherwise "2E" could swallow the start of a reference like E5. + var save = _pos; + _pos++; + if (_pos < _s.Length && (_s[_pos] == '+' || _s[_pos] == '-')) _pos++; + if (_pos < _s.Length && _s[_pos] >= '0' && _s[_pos] <= '9') + { + while (_pos < _s.Length && _s[_pos] >= '0' && _s[_pos] <= '9') _pos++; + } + else _pos = save; + } + return double.TryParse(_s.Substring(start, _pos - start), NumberStyles.Float, CultureInfo.InvariantCulture, out var v) ? v : Fail(); + } + + private string ReadIdentifier() + { + var start = _pos; + while (_pos < _s.Length && (IsLetter(_s[_pos]) || (_s[_pos] >= '0' && _s[_pos] <= '9') || _s[_pos] == '_' || _s[_pos] == '.' || _s[_pos] == '$')) _pos++; + return _s.Substring(start, _pos - start); + } + + private double ParseFunctionCall(string name) + { + Function fn; + switch (name.ToUpperInvariant()) + { + case "SUM": fn = Function.Sum; break; + case "AVERAGE": fn = Function.Average; break; + case "COUNT": fn = Function.Count; break; + case "COUNTA": fn = Function.CountA; break; + case "MIN": fn = Function.Min; break; + case "MAX": fn = Function.Max; break; + case "ROUND": fn = Function.Round; break; + case "ABS": fn = Function.Abs; break; + default: return Fail(); + } + + // Aggregate state; the function decides which fields matter. + double sum = 0, min = double.MaxValue, max = double.MinValue; + var count = 0; + var argCount = 0; + double arg0 = 0, arg1 = 0; + + SkipSpaces(); + if (_pos < _s.Length && _s[_pos] == ')') return Fail(); // Excel rejects empty argument lists too. + while (!Failed) + { + argCount++; + if (TryParseRangeArgument(out var range)) + { + if (fn == Function.Round || fn == Function.Abs) + { + // Scalar functions take a single cell, read like an arithmetic operand. + if (range.RowCount != 1 || range.ColumnCount != 1) return Fail(); + var scalar = CellValueForArithmetic(range.FromRow, range.FromColumn); + if (argCount == 1) arg0 = scalar; else arg1 = scalar; + } + else if (!Aggregate(range, fn, ref sum, ref min, ref max, ref count)) return Fail(); + } + else + { + var v = ParseExpression(); + if (Failed) return 0; + // A literal or expression argument is a number for every function (SUM(1, A1) adds 1). + sum += v; + count++; + if (v < min) min = v; + if (v > max) max = v; + if (argCount == 1) arg0 = v; else arg1 = v; + } + SkipSpaces(); + if (_pos >= _s.Length) return Fail(); + if (_s[_pos] == ',') { _pos++; continue; } + if (_s[_pos] == ')') { _pos++; break; } + return Fail(); + } + if (Failed) return 0; + + switch (fn) + { + case Function.Sum: return sum; + case Function.Average: return count == 0 ? Fail() : sum / count; // AVERAGE of nothing is #DIV/0!. + case Function.Count: return count; + case Function.CountA: return count; + case Function.Min: return count == 0 ? 0 : min; // Excel returns 0 for MIN/MAX over empty cells. + case Function.Max: return count == 0 ? 0 : max; + case Function.Abs: return argCount == 1 ? Math.Abs(arg0) : Fail(); + default: return argCount == 2 ? Round(arg0, arg1) : Fail(); + } + } + + /// Folds the cells of a range into the aggregate state. False when a cell is unknown or not yet written. + private bool Aggregate(RangeReference range, Function fn, ref double sum, ref double min, ref double max, ref int count) + { + if (!IsWritten(range.ToRow, range.ToColumn)) return false; + for (var r = range.FromRow; r <= range.ToRow; r++) + { + for (var c = range.FromColumn; c <= range.ToColumn; c++) + { + var state = _values.Get(r, c, out var v); + switch (state) + { + case FormulaCellState.Unknown: return false; + case FormulaCellState.Empty: continue; + case FormulaCellState.Text: + if (fn == Function.CountA) count++; + continue; + default: + count++; + sum += v; + if (v < min) min = v; + if (v > max) max = v; + continue; + } + } + } + return true; + } + + /// + /// Parses an argument that is a bare reference (C2:C101, $B$7, 'Sheet'!A1:A9) followed + /// by , or ). Anything else (an expression such as B7*2) is left for the scalar path. + /// + private bool TryParseRangeArgument(out RangeReference range) + { + var start = _pos; + SkipSpaces(); + if (TryParseReference(out range, out _)) + { + SkipSpaces(); + if (_pos < _s.Length && (_s[_pos] == ',' || _s[_pos] == ')')) return true; + } + _pos = start; // Not a reference argument: rewind and let the caller parse an expression. + return false; + } + + /// + /// Parses [sheet!]cell[:cell]. A sheet qualifier must name the current sheet; another sheet's + /// values are not available to a streaming writer, so that fails. + /// + private bool TryParseReference(out RangeReference range, out bool isRange) + { + range = default; + isRange = false; + var start = _pos; + + if (_pos < _s.Length && _s[_pos] == '\'') + { + // Quoted sheet name; an embedded apostrophe is doubled. + _pos++; + var sb = new System.Text.StringBuilder(); + while (true) + { + if (_pos >= _s.Length) return false; + if (_s[_pos] == '\'') + { + if (_pos + 1 < _s.Length && _s[_pos + 1] == '\'') { sb.Append('\''); _pos += 2; continue; } + _pos++; + break; + } + sb.Append(_s[_pos++]); + } + if (_pos >= _s.Length || _s[_pos] != '!') return false; + _pos++; + if (!IsCurrentSheet(sb.ToString())) return false; + } + else + { + // Unquoted: "Sheet1!A1". Only when a '!' follows the identifier; otherwise it is the cell itself. + var save = _pos; + var ident = ReadIdentifier(); + if (_pos < _s.Length && _s[_pos] == '!') + { + _pos++; + if (!IsCurrentSheet(ident)) return false; + } + else _pos = save; + } + + if (!TryParseCell(out var r1, out var c1)) { _pos = start; return false; } + var r2 = r1; + var c2 = c1; + if (_pos < _s.Length && _s[_pos] == ':') + { + _pos++; + if (!TryParseCell(out r2, out c2)) { _pos = start; return false; } + isRange = true; + } + range = new RangeReference(Math.Min(r1, r2), Math.Min(c1, c2), Math.Max(r1, r2), Math.Max(c1, c2)); + return true; + } + + /// Parses B7 or $B$7 (case-insensitive) into a row and column. + private bool TryParseCell(out int row, out int column) + { + row = column = 0; + if (_pos < _s.Length && _s[_pos] == '$') _pos++; + var letters = 0; + while (_pos < _s.Length && IsLetter(_s[_pos])) + { + column = column * 26 + (char.ToUpperInvariant(_s[_pos]) - 'A' + 1); + letters++; + _pos++; + } + if (letters == 0 || letters > 3 || column > CellReference.MaxColumns) return false; + if (_pos < _s.Length && _s[_pos] == '$') _pos++; + var digits = 0; + while (_pos < _s.Length && _s[_pos] >= '0' && _s[_pos] <= '9') + { + row = row * 10 + (_s[_pos] - '0'); + digits++; + _pos++; + if (row > CellReference.MaxRows) return false; + } + if (digits == 0 || row < 1) return false; + // "A1B" would be a defined name, not a reference. + if (_pos < _s.Length && (IsLetter(_s[_pos]) || _s[_pos] == '_' || _s[_pos] == '.')) return false; + return true; + } + + /// Value of one cell used as an arithmetic operand: numbers as-is, empty as 0, text and unknown fail. + private double CellValueForArithmetic(int row, int column) + { + if (!IsWritten(row, column)) return Fail(); + switch (_values.Get(row, column, out var v)) + { + case FormulaCellState.Number: return v; + case FormulaCellState.Empty: return 0; + default: return Fail(); // Text in arithmetic is #VALUE!; a boolean would be coerced but is not tracked. + } + } + + /// True when the cell precedes the one being written: a row above, or the same row to the left. + private bool IsWritten(int row, int column) => row < _row || (row == _row && column < _column); + + private bool IsCurrentSheet(string name) => string.Equals(name, _sheetName, StringComparison.OrdinalIgnoreCase); + + private void SkipSpaces() + { + while (_pos < _s.Length && (_s[_pos] == ' ' || _s[_pos] == '\t' || _s[_pos] == '\n' || _s[_pos] == '\r')) _pos++; + } + + private static bool IsLetter(char c) => (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z'); + + /// Excel's ROUND: half away from zero, negative digits round to tens, hundreds... + private static double Round(double value, double digits) + { + var n = (int)Math.Truncate(digits); + if (n >= 0) return n > 15 ? value : Math.Round(value, n, MidpointRounding.AwayFromZero); + var scale = Math.Pow(10, -n); + return Math.Round(value / scale, MidpointRounding.AwayFromZero) * scale; + } + } + } +} diff --git a/src/SheetStream/Internal/FormulaValueStore.cs b/src/SheetStream/Internal/FormulaValueStore.cs new file mode 100644 index 0000000..0c5b1ca --- /dev/null +++ b/src/SheetStream/Internal/FormulaValueStore.cs @@ -0,0 +1,133 @@ +using System; + +namespace SheetStream.Internal +{ + /// What a recorded cell holds, as far as the formula evaluator is concerned. + internal enum FormulaCellState + { + /// Nothing was written to the cell (or only a style). + Empty, + + /// A number, date, time span or a formula whose numeric result is known. + Number, + + /// Text, a boolean, an error or a formula with a text/boolean result: present but not a number. + Text, + + /// Content whose value is not known (a formula without a cached result, a template cell). + Unknown, + } + + /// + /// Compact record of the values written to a worksheet so far, kept only when the sheet caches formula + /// results. Each column is a set of fixed-size pages of double allocated on demand, so a numeric + /// cell in a dense export costs 8 bytes, while a single cell at row 1,000,000 costs one page (8 KB) + /// rather than an 8 MB array. That bound matters in edit mode, where the rows come from a file the + /// application did not write. The three non-numeric states are stored in the same slot as NaN values + /// with distinct payloads, which is why no second array is needed: a real NaN is never recorded because + /// the writer turns NaN and infinities into #NUM! error cells. + /// + internal sealed class FormulaValueStore + { + // NaN payloads for the non-numeric states. Compared by bit pattern since NaN != NaN. + private const long EmptyBits = unchecked((long)0x7FF8000000000001UL); + private const long TextBits = unchecked((long)0x7FF8000000000002UL); + private const long UnknownBits = unchecked((long)0x7FF8000000000003UL); + private static readonly double EmptyValue = BitConverter.Int64BitsToDouble(EmptyBits); + private static readonly double TextValue = BitConverter.Int64BitsToDouble(TextBits); + private static readonly double UnknownValue = BitConverter.Int64BitsToDouble(UnknownBits); + + // 1,024 rows per page: 8 KB, small enough that a stray far-away cell is cheap and large enough that a + // dense column of a million rows needs only ~1,000 page references. + private const int PageShift = 10; + private const int PageSize = 1 << PageShift; + private const int PageMask = PageSize - 1; + + // Column -> pages of that column -> values. Both outer levels grow by doubling, capped at the sheet limits. + private double[]?[]?[] _columns = new double[]?[]?[32]; + + // Rows whose content came from somewhere the writer did not see (copied template rows). A bit per row. + private ulong[] _unknownRows = new ulong[0]; + + /// Records a numeric value (numbers, dates and durations as serials, cached formula results). + public void SetNumber(int row, int column, double value) => Slot(row, column) = value; + + /// Records that the cell holds something that is not a number (text, boolean, error). + public void SetText(int row, int column) => Slot(row, column) = TextValue; + + /// Records that the cell has content whose value cannot be known here. + public void SetUnknown(int row, int column) => Slot(row, column) = UnknownValue; + + /// Returns the state of a cell and, for , its value. + public FormulaCellState Get(int row, int column, out double value) + { + value = 0; + if (IsRowUnknown(row)) return FormulaCellState.Unknown; + if (column >= _columns.Length) return FormulaCellState.Empty; + var pages = _columns[column]; + var pageIndex = row >> PageShift; + if (pages == null || pageIndex >= pages.Length) return FormulaCellState.Empty; + var page = pages[pageIndex]; + if (page == null) return FormulaCellState.Empty; + var v = page[row & PageMask]; + if (!double.IsNaN(v)) + { + value = v; + return FormulaCellState.Number; + } + var bits = BitConverter.DoubleToInt64Bits(v); + return bits == TextBits ? FormulaCellState.Text : bits == UnknownBits ? FormulaCellState.Unknown : FormulaCellState.Empty; + } + + /// Marks a whole row as opaque: a template row copied as XML, whose cells were never seen. + public void MarkRowUnknown(int row) + { + var word = row >> 6; + if (word >= _unknownRows.Length) + { + var grown = new ulong[Math.Max(word + 1, _unknownRows.Length * 2)]; + Array.Copy(_unknownRows, grown, _unknownRows.Length); + _unknownRows = grown; + } + _unknownRows[word] |= 1UL << (row & 63); + } + + /// True when was called for the row. + public bool IsRowUnknown(int row) + { + var word = row >> 6; + return word < _unknownRows.Length && (_unknownRows[word] & (1UL << (row & 63))) != 0; + } + + /// Returns the slot for a cell, growing the column list, the column's page list and the page as needed. + private ref double Slot(int row, int column) + { + if (column >= _columns.Length) + { + var grown = new double[]?[]?[Math.Min(CellReference.MaxColumns + 1, Math.Max(column + 1, _columns.Length * 2))]; + Array.Copy(_columns, grown, _columns.Length); + _columns = grown; + } + + var pages = _columns[column]; + var pageIndex = row >> PageShift; + if (pages == null || pageIndex >= pages.Length) + { + var oldLength = pages?.Length ?? 0; + var grown = new double[]?[Math.Min((CellReference.MaxRows >> PageShift) + 1, Math.Max(pageIndex + 1, Math.Max(oldLength * 2, 4)))]; + if (pages != null) Array.Copy(pages, grown, oldLength); + _columns[column] = pages = grown; + } + + var page = pages[pageIndex]; + if (page == null) + { + // A fresh page is filled with the Empty sentinel because a default 0.0 would read as a legitimate number. + page = new double[PageSize]; + for (var i = 0; i < page.Length; i++) page[i] = EmptyValue; + pages[pageIndex] = page; + } + return ref page[row & PageMask]; + } + } +} diff --git a/src/SheetStream/Internal/OpenXmlNamespaces.cs b/src/SheetStream/Internal/OpenXmlNamespaces.cs index 5cbff18..a84fcdb 100644 --- a/src/SheetStream/Internal/OpenXmlNamespaces.cs +++ b/src/SheetStream/Internal/OpenXmlNamespaces.cs @@ -39,6 +39,9 @@ internal static class OpenXmlNamespaces public const string RelChart = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/chart"; public const string RelComments = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/comments"; public const string RelVmlDrawing = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/vmlDrawing"; + public const string RelPivotTable = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/pivotTable"; + public const string RelPivotCacheDefinition = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/pivotCacheDefinition"; + public const string RelPivotCacheRecords = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/pivotCacheRecords"; public const string RelCoreProperties = "http://schemas.openxmlformats.org/package/2006/relationships/metadata/core-properties"; public const string RelExtendedProperties = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/extended-properties"; @@ -54,6 +57,9 @@ internal static class OpenXmlNamespaces public const string CtChart = "application/vnd.openxmlformats-officedocument.drawingml.chart+xml"; public const string CtComments = "application/vnd.openxmlformats-officedocument.spreadsheetml.comments+xml"; public const string CtVmlDrawing = "application/vnd.openxmlformats-officedocument.vmlDrawing"; + public const string CtPivotTable = "application/vnd.openxmlformats-officedocument.spreadsheetml.pivotTable+xml"; + public const string CtPivotCacheDefinition = "application/vnd.openxmlformats-officedocument.spreadsheetml.pivotCacheDefinition+xml"; + public const string CtPivotCacheRecords = "application/vnd.openxmlformats-officedocument.spreadsheetml.pivotCacheRecords+xml"; public const string CtCoreProperties = "application/vnd.openxmlformats-package.core-properties+xml"; public const string CtExtendedProperties = "application/vnd.openxmlformats-officedocument.extended-properties+xml"; } diff --git a/src/SheetStream/Reading/ExcelReader.Async.cs b/src/SheetStream/Reading/ExcelReader.Async.cs new file mode 100644 index 0000000..db15f02 --- /dev/null +++ b/src/SheetStream/Reading/ExcelReader.Async.cs @@ -0,0 +1,370 @@ +#if NET8_0_OR_GREATER +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Data; +using System.Diagnostics.CodeAnalysis; +using System.IO; +using System.IO.Compression; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Xml; + +namespace SheetStream.Reading +{ + /// + /// Asynchronous reading: the counterpart of Worksheet.LoadFromAsyncEnumerable. An upload can be + /// buffered with and its rows pulled with + /// without ever blocking a request thread on I/O. + /// Only available on .NET 8 and later; netstandard2.0 has no built-in . + /// + public sealed partial class ExcelReader + { + /// + /// How much of a non-seekable stream keeps in + /// memory before spilling the rest into a temporary file: 64 MB. Below the limit the whole upload sits + /// in a ; above it the bytes already buffered plus everything that follows go + /// to a file opened with , so a multi-hundred-megabyte upload + /// never doubles the process' managed heap and the file disappears when the reader is disposed. + /// + internal const long InMemoryBufferLimit = 64L * 1024 * 1024; + + /// + /// Opens a workbook from a stream, buffering asynchronously when the stream cannot seek. A seekable stream + /// (a or ) is wrapped directly, exactly like + /// . A non-seekable one (an ASP.NET Core request body, an + /// HttpClient response) must be copied first because the zip directory sits at the end of the + /// file; the copy uses CopyToAsync-style reads so no thread is blocked on the network, and it + /// stays in memory up to (64 MB) before spilling into a temporary file + /// that is deleted with the reader. + /// + /// + /// Once the bytes are in place the package directory, workbook, shared strings and styles are parsed + /// synchronously: has no asynchronous API and those parts are small compared + /// with the sheets. The row data itself is read asynchronously by . + /// + /// The .xlsx content. + /// True to leave open when the reader is disposed (or, for a non-seekable stream, after it has been copied). + /// Cancels the copy of a non-seekable stream. + public static Task OpenAsync(Stream stream, bool leaveOpen = false, CancellationToken cancellationToken = default) + { + if (stream == null) throw new ArgumentNullException(nameof(stream)); + if (stream.CanSeek) return Task.FromResult(new ExcelReader(stream, leaveOpen)); + return OpenBufferedAsync(stream, leaveOpen, cancellationToken); + } + + private static async Task OpenBufferedAsync(Stream stream, bool leaveOpen, CancellationToken cancellationToken) + { + var copy = await BufferAsync(stream, InMemoryBufferLimit, cancellationToken).ConfigureAwait(false); + try + { + // Same order as the synchronous Open: the source is released before the package is parsed. + if (!leaveOpen) await stream.DisposeAsync().ConfigureAwait(false); + return new ExcelReader(copy, false); + } + catch + { + await copy.DisposeAsync().ConfigureAwait(false); // Also deletes a spilled temp file. + throw; + } + } + + /// + /// Opens a workbook file for asynchronous reading. The file is opened for overlapped I/O + /// () so that + /// pulls sheet bytes with true asynchronous reads instead of thread-pool emulation. Opening itself is not + /// fully asynchronous: the zip directory, workbook, shared strings and styles are read synchronously + /// because has no asynchronous API; the returned task only surfaces I/O errors + /// (a missing file, say) the way an async caller expects. Like , the file is + /// shared for reading so Excel can keep it open. + /// + public static Task OpenAsync(string path, CancellationToken cancellationToken = default) + { + if (path == null) throw new ArgumentNullException(nameof(path)); + if (cancellationToken.IsCancellationRequested) return Task.FromCanceled(cancellationToken); + try + { + var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, 1 << 16, FileOptions.Asynchronous); + try { return Task.FromResult(new ExcelReader(fs, false)); } + catch { fs.Dispose(); throw; } + } + catch (Exception ex) when (!(ex is ArgumentException)) + { + // Argument errors are programming mistakes and throw synchronously; everything else (file not + // found, access denied, a corrupt package) is reported through the task like any async I/O call. + return Task.FromException(ex); + } + } + + /// + /// Copies a non-seekable stream into a seekable one positioned at zero: a + /// while the content fits in bytes, otherwise a temporary file opened + /// with . Internal so tests can exercise the spill with a small limit. + /// + internal static async Task BufferAsync(Stream source, long memoryLimit, CancellationToken cancellationToken) + { + var memory = new MemoryStream(); + // 80 KB keeps the scratch buffer below the large-object threshold; the pool makes it free to rent. + var buffer = ArrayPool.Shared.Rent(81920); + try + { + int read; + while ((read = await source.ReadAsync(buffer.AsMemory(), cancellationToken).ConfigureAwait(false)) > 0) + { + if (memory.Length + read > memoryLimit) + { + return await SpillToFileAsync(memory, buffer.AsMemory(0, read), source, cancellationToken).ConfigureAwait(false); + } + memory.Write(buffer, 0, read); + } + memory.Position = 0; + return memory; + } + finally + { + ArrayPool.Shared.Return(buffer); + } + } + + /// Moves what is already buffered plus the rest of into a self-deleting temp file. + private static async Task SpillToFileAsync(MemoryStream buffered, ReadOnlyMemory pending, Stream source, CancellationToken cancellationToken) + { + // GetRandomFileName + CreateNew instead of GetTempFileName: the latter caps at 65,535 files per + // folder and creates the file twice. DeleteOnClose ties the file's life to the reader's. + var path = Path.Combine(Path.GetTempPath(), "sheetstream-" + Path.GetRandomFileName()); + var file = new FileStream(path, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, 1 << 16, FileOptions.Asynchronous | FileOptions.DeleteOnClose); + try + { + await file.WriteAsync(buffered.GetBuffer().AsMemory(0, (int)buffered.Length), cancellationToken).ConfigureAwait(false); + await file.WriteAsync(pending, cancellationToken).ConfigureAwait(false); + await source.CopyToAsync(file, cancellationToken).ConfigureAwait(false); + await file.FlushAsync(cancellationToken).ConfigureAwait(false); + file.Position = 0; + return file; + } + catch + { + await file.DisposeAsync().ConfigureAwait(false); + throw; + } + } + + // ------------------------------------------------------------------ rows + + /// + /// Streams the rows of a sheet by name (case-insensitive) without blocking a thread: the sheet XML is + /// parsed with the asynchronous methods and cancellation is checked before every + /// row. Values are identical to ; the two share the cell parsing code. + /// Breaking out of the await foreach early disposes the XML reader and the zip entry stream. + /// + /// + /// + /// using var reader = await ExcelReader.OpenAsync(Request.Body, cancellationToken: ct); + /// await foreach (var row in reader.ReadRowsAsync("Products", ct)) + /// Console.WriteLine($"{row.GetString(0)} = {row.GetDouble(2)}"); + /// + /// + public IAsyncEnumerable ReadRowsAsync(string sheetName, CancellationToken cancellationToken = default) + => ReadRowsAsync(FindSheet(sheetName), cancellationToken); + + /// Streams the rows of a sheet by zero-based position; see . + public IAsyncEnumerable ReadRowsAsync(int sheetIndex, CancellationToken cancellationToken = default) + => ReadRowsCoreAsync(SheetEntry(sheetIndex), cancellationToken); + + /// + /// The asynchronous twin of : same cursor discipline, same helpers for placing + /// and typing cells, with every Read replaced by its awaited form. The token passed here is + /// combined by the compiler with one supplied later through WithCancellation. + /// + private async IAsyncEnumerable ReadRowsCoreAsync(ZipArchiveEntry entry, [EnumeratorCancellation] CancellationToken cancellationToken) + { + // A plain using is enough: XmlReader is IDisposable only, and the async iterator runs this finally + // block both on normal completion and when the consumer disposes the enumerator early. + using (var xr = XmlReader.Create(entry.Open(), SheetSettings(async: true))) + { + var lastRow = 0; + var values = new List(); + while (await xr.ReadAsync().ConfigureAwait(false)) + { + if (xr.NodeType != XmlNodeType.Element || xr.LocalName != "row") continue; + // Between rows is the one place cancellation is honoured: a row is parsed whole or not at all. + cancellationToken.ThrowIfCancellationRequested(); + var rowIndex = ParseInt(xr.GetAttribute("r")) ?? lastRow + 1; + lastRow = rowIndex; + values.Clear(); + if (!xr.IsEmptyElement) + { + var depth = xr.Depth; + var nextColumn = 1; + await xr.ReadAsync().ConfigureAwait(false); + while (!xr.EOF && !(xr.NodeType == XmlNodeType.EndElement && xr.Depth == depth)) + { + if (xr.NodeType == XmlNodeType.Element && xr.LocalName == "c") + { + var reference = xr.GetAttribute("r"); + var column = reference != null ? ColumnOf(reference) : nextColumn; + nextColumn = column + 1; + var value = await ReadCellAsync(xr).ConfigureAwait(false); // Leaves the reader on or on an empty . + await xr.ReadAsync().ConfigureAwait(false); + if (value != null) PlaceCell(values, column, value); + continue; + } + await xr.ReadAsync().ConfigureAwait(false); + } + } + yield return new ExcelRow(rowIndex, values.ToArray()); + } + } + } + + /// + /// Asynchronous twin of : collects the raw parts of a c element and hands them + /// to the shared . Returns a because the XML buffer + /// usually already holds the cell, so most calls complete synchronously without allocating a task. + /// + private async ValueTask ReadCellAsync(XmlReader xr) + { + var type = xr.GetAttribute("t"); + var styleIndex = ParseInt(xr.GetAttribute("s")) ?? 0; + if (xr.IsEmptyElement) return null; + + string? v = null; + string? inline = null; + var depth = xr.Depth; + await xr.ReadAsync().ConfigureAwait(false); + while (!xr.EOF && !(xr.NodeType == XmlNodeType.EndElement && xr.Depth == depth)) + { + if (xr.NodeType == XmlNodeType.Element) + { + if (xr.LocalName == "v") + { + if (xr.IsEmptyElement) { v = ""; await xr.ReadAsync().ConfigureAwait(false); } + else v = await xr.ReadElementContentAsStringAsync().ConfigureAwait(false); // Advances past . + continue; + } + if (xr.LocalName == "is") + { + if (xr.IsEmptyElement) inline = ""; + else inline = await ReadRichTextAsync(xr).ConfigureAwait(false); // Leaves the reader on . + await xr.ReadAsync().ConfigureAwait(false); + continue; + } + await xr.SkipAsync().ConfigureAwait(false); // and anything else: skip the whole element. + continue; + } + await xr.ReadAsync().ConfigureAwait(false); + } + + return CellValue(type, styleIndex, v, inline); + } + + /// Asynchronous twin of for inline strings; same contract, same decoding. + private static async ValueTask ReadRichTextAsync(XmlReader xr) + { + var sb = new StringBuilder(); + var depth = xr.Depth; + await xr.ReadAsync().ConfigureAwait(false); + while (!xr.EOF && !(xr.NodeType == XmlNodeType.EndElement && xr.Depth == depth)) + { + if (xr.NodeType == XmlNodeType.Element) + { + if (xr.LocalName == "rPh") { await xr.SkipAsync().ConfigureAwait(false); continue; } + if (xr.LocalName == "t") + { + if (xr.IsEmptyElement) await xr.ReadAsync().ConfigureAwait(false); + else sb.Append(await xr.ReadElementContentAsStringAsync().ConfigureAwait(false)); // Advances past . + continue; + } + } + await xr.ReadAsync().ConfigureAwait(false); + } + return Decode(sb.ToString()); // The reader is now on the container's end element. + } + + // ------------------------------------------------------------------ objects and tables + + /// + /// Maps rows to objects asynchronously, with the rules of : the first row is + /// the header, headers match members case-insensitively, a value that cannot be converted throws a + /// naming the row and column, and blank rows are skipped. A mapper + /// registered in takes precedence over reflection. + /// + /// + /// + /// await foreach (var product in reader.ReadAsync<Product>("Products", ct)) + /// await db.Products.AddAsync(product, ct); + /// + /// + public IAsyncEnumerable ReadAsync<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicProperties | DynamicallyAccessedMemberTypes.PublicFields)] T>(string sheetName, CancellationToken cancellationToken = default) where T : new() + { + if (SheetMappers.TryGet(out var mapper)) return ReadAsync(sheetName, mapper, cancellationToken); + return ReadReflectedAsync(FindSheet(sheetName), cancellationToken); // Wrong sheet name fails here, not on the first MoveNextAsync. + } + + /// + /// Maps rows to objects asynchronously through an explicit , like + /// . No reflection is involved, so this is the overload + /// for trimmed and Native AOT applications. + /// + public IAsyncEnumerable ReadAsync(string sheetName, ISheetRowMapper mapper, CancellationToken cancellationToken = default) + { + if (mapper == null) throw new ArgumentNullException(nameof(mapper)); + if (!mapper.CanRead) throw new NotSupportedException($"The mapper for {typeof(T).Name} cannot create instances (the type has no parameterless constructor), so it can only be used for writing."); + return ReadMappedAsync(FindSheet(sheetName), mapper, cancellationToken); + } + + private async IAsyncEnumerable ReadReflectedAsync<[DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicProperties | DynamicallyAccessedMemberTypes.PublicFields)] T>(int sheetIndex, [EnumeratorCancellation] CancellationToken cancellationToken) where T : new() + { + Binding[]? bindings = null; + await foreach (var row in ReadRowsAsync(sheetIndex, cancellationToken).ConfigureAwait(false)) + { + if (bindings == null) + { + bindings = Bind(typeof(T), row); + continue; + } + if (MapRow(row, bindings, out T item)) yield return item; + } + } + + private async IAsyncEnumerable ReadMappedAsync(int sheetIndex, ISheetRowMapper mapper, [EnumeratorCancellation] CancellationToken cancellationToken) + { + int[]? columns = null; + await foreach (var row in ReadRowsAsync(sheetIndex, cancellationToken).ConfigureAwait(false)) + { + if (columns == null) + { + columns = BindColumns(mapper, row); + continue; + } + if (MapRow(row, columns, mapper, out var item)) yield return item; + } + } + + /// + /// Reads a sheet into a asynchronously, with the column rules of + /// . The table is built in memory, so the gain over the + /// synchronous call is only that the sheet bytes are read without blocking a thread. + /// + public async Task ReadDataTableAsync(string sheetName, bool firstRowIsHeader = true, CancellationToken cancellationToken = default) + { + var table = new DataTable(sheetName); + var first = true; + await foreach (var row in ReadRowsAsync(sheetName, cancellationToken).ConfigureAwait(false)) + { + if (first && firstRowIsHeader) + { + first = false; + AddHeaderColumns(table, row); + continue; + } + first = false; + AddDataRow(table, row); + } + return table; + } + } +} +#endif diff --git a/src/SheetStream/Reading/ExcelReader.Mapping.cs b/src/SheetStream/Reading/ExcelReader.Mapping.cs index b7120f0..2b945e8 100644 --- a/src/SheetStream/Reading/ExcelReader.Mapping.cs +++ b/src/SheetStream/Reading/ExcelReader.Mapping.cs @@ -35,26 +35,35 @@ private IEnumerable ReadMapped(string sheetName, ISheetRowMapper mapper continue; } - var item = mapper.Create(); - var any = false; - for (var sheetColumn = 0; sheetColumn < columns.Length; sheetColumn++) + // Blank rows are skipped, the same rule the reflection reader applies. + if (MapRow(row, columns, mapper, out var item)) yield return item; + } + } + + /// + /// Fills a mapper-created from one data row. Returns false for a blank row (no + /// mapped column has a value). Shared by the sync and async readers. + /// + private static bool MapRow(ExcelRow row, int[] columns, ISheetRowMapper mapper, out T item) + { + item = mapper.Create(); + var any = false; + for (var sheetColumn = 0; sheetColumn < columns.Length; sheetColumn++) + { + var column = columns[sheetColumn]; + if (column < 0) continue; + var raw = row[sheetColumn]; + if (raw == null) continue; + any = true; + object? converted; + try { converted = Convert(raw, mapper.MemberType(column)); } + catch (Exception ex) when (ex is FormatException || ex is InvalidCastException || ex is OverflowException || ex is ArgumentException) { - var column = columns[sheetColumn]; - if (column < 0) continue; - var raw = row[sheetColumn]; - if (raw == null) continue; - any = true; - object? converted; - try { converted = Convert(raw, mapper.MemberType(column)); } - catch (Exception ex) when (ex is FormatException || ex is InvalidCastException || ex is OverflowException || ex is ArgumentException) - { - throw new FormatException($"Row {row.Index}, column {CellReference.ColumnLetters(sheetColumn + 1)} ('{mapper.Headers[column]}'): cannot convert '{raw}' to {mapper.MemberType(column).Name}.", ex); - } - mapper.SetValue(ref item, column, converted); + throw new FormatException($"Row {row.Index}, column {CellReference.ColumnLetters(sheetColumn + 1)} ('{mapper.Headers[column]}'): cannot convert '{raw}' to {mapper.MemberType(column).Name}.", ex); } - // Blank rows are skipped, the same rule the reflection reader applies. - if (any) yield return item; + mapper.SetValue(ref item, column, converted); } + return any; } private static int[] BindColumns(ISheetRowMapper mapper, ExcelRow header) diff --git a/src/SheetStream/Reading/ExcelReader.cs b/src/SheetStream/Reading/ExcelReader.cs index ab9ffaf..b3a0e4f 100644 --- a/src/SheetStream/Reading/ExcelReader.cs +++ b/src/SheetStream/Reading/ExcelReader.cs @@ -171,12 +171,14 @@ public IReadOnlyList SheetNames public IEnumerable ReadRows(string sheetName) => ReadRows(FindSheet(sheetName)); /// Streams the rows of a sheet by zero-based position. - public IEnumerable ReadRows(int sheetIndex) + public IEnumerable ReadRows(int sheetIndex) => ReadRowsCore(SheetEntry(sheetIndex)); + + /// Resolves a sheet position to its zip entry, failing eagerly (not on the first MoveNext) for a bad index or a damaged package. + private ZipArchiveEntry SheetEntry(int sheetIndex) { if (sheetIndex < 0 || sheetIndex >= _sheets.Count) throw new ArgumentOutOfRangeException(nameof(sheetIndex), sheetIndex, $"The workbook has {_sheets.Count} sheet(s)."); - var entry = _zip.GetEntry(_sheets[sheetIndex].Part) + return _zip.GetEntry(_sheets[sheetIndex].Part) ?? throw new InvalidDataException($"Sheet '{_sheets[sheetIndex].Name}' points at missing part '{_sheets[sheetIndex].Part}'."); - return ReadRowsCore(entry); } /// @@ -193,27 +195,39 @@ public DataTable ReadDataTable(string sheetName, bool firstRowIsHeader = true) if (first && firstRowIsHeader) { first = false; - var used = new HashSet(StringComparer.OrdinalIgnoreCase); - for (var i = 0; i < row.Values.Count; i++) - { - var name = row.GetString(i); - if (string.IsNullOrWhiteSpace(name)) name = CellReference.ColumnLetters(i + 1); - var unique = name!; - for (var n = 2; !used.Add(unique); n++) unique = name + n.ToString(CultureInfo.InvariantCulture); - table.Columns.Add(unique, typeof(object)); - } + AddHeaderColumns(table, row); continue; } first = false; - while (table.Columns.Count < row.Values.Count) - table.Columns.Add(CellReference.ColumnLetters(table.Columns.Count + 1), typeof(object)); - var values = new object[table.Columns.Count]; - for (var i = 0; i < values.Length; i++) values[i] = row[i] ?? DBNull.Value; - table.Rows.Add(values); + AddDataRow(table, row); } return table; } + /// Turns the header row into table columns: blank headers get the column letter, duplicates a numeric suffix. + private static void AddHeaderColumns(DataTable table, ExcelRow row) + { + var used = new HashSet(StringComparer.OrdinalIgnoreCase); + for (var i = 0; i < row.Values.Count; i++) + { + var name = row.GetString(i); + if (string.IsNullOrWhiteSpace(name)) name = CellReference.ColumnLetters(i + 1); + var unique = name!; + for (var n = 2; !used.Add(unique); n++) unique = name + n.ToString(CultureInfo.InvariantCulture); + table.Columns.Add(unique, typeof(object)); + } + } + + /// Appends a sheet row to the table, growing the column list when a row is wider than the header. + private static void AddDataRow(DataTable table, ExcelRow row) + { + while (table.Columns.Count < row.Values.Count) + table.Columns.Add(CellReference.ColumnLetters(table.Columns.Count + 1), typeof(object)); + var values = new object[table.Columns.Count]; + for (var i = 0; i < values.Length; i++) values[i] = row[i] ?? DBNull.Value; + table.Rows.Add(values); + } + /// /// Maps rows to objects. The first row is the header; each header is matched (case-insensitively) to a /// writable property or field by name, [DisplayName] or member name. @@ -232,24 +246,33 @@ public DataTable ReadDataTable(string sheetName, bool firstRowIsHeader = true) bindings = Bind(typeof(T), row); continue; } - var item = new T(); - object boxed = item!; - var any = false; - foreach (var b in bindings) + if (MapRow(row, bindings, out T item)) yield return item; + } + } + + /// + /// Fills a new from one data row through reflection bindings. Returns false for a + /// blank row (no bound column has a value), which callers skip. Shared by the sync and async readers. + /// + private static bool MapRow(ExcelRow row, Binding[] bindings, out T item) where T : new() + { + object boxed = new T()!; + var any = false; + foreach (var b in bindings) + { + var raw = row[b.Column]; + if (raw == null) continue; + any = true; + object? converted; + try { converted = Convert(raw, b.Type); } + catch (Exception ex) when (ex is FormatException || ex is InvalidCastException || ex is OverflowException || ex is ArgumentException) { - var raw = row[b.Column]; - if (raw == null) continue; - any = true; - object? converted; - try { converted = Convert(raw, b.Type); } - catch (Exception ex) when (ex is FormatException || ex is InvalidCastException || ex is OverflowException || ex is ArgumentException) - { - throw new FormatException($"Row {row.Index}, column {CellReference.ColumnLetters(b.Column + 1)} ('{b.Header}'): cannot convert '{raw}' to {b.Type.Name}.", ex); - } - b.Set(boxed, converted); + throw new FormatException($"Row {row.Index}, column {CellReference.ColumnLetters(b.Column + 1)} ('{b.Header}'): cannot convert '{raw}' to {b.Type.Name}.", ex); } - if (any) yield return (T)boxed; + b.Set(boxed, converted); } + item = (T)boxed; + return any; } /// @@ -432,9 +455,17 @@ private static string Resolve(string folder, string target) // ------------------------------------------------------------------ rows and cells + /// + /// Settings for parsing a sheet part. DTDs are never resolved (untrusted input), and the reader owns the + /// zip entry stream so disposing it closes both. enables the *Async + /// methods used by ReadRowsAsync; the sync reader leaves it off because it costs a little per call. + /// + private static XmlReaderSettings SheetSettings(bool async) + => new XmlReaderSettings { DtdProcessing = DtdProcessing.Prohibit, XmlResolver = null, CloseInput = true, IgnoreComments = true, Async = async }; + private IEnumerable ReadRowsCore(ZipArchiveEntry entry) { - using (var xr = XmlReader.Create(entry.Open(), new XmlReaderSettings { DtdProcessing = DtdProcessing.Prohibit, XmlResolver = null, CloseInput = true, IgnoreComments = true })) + using (var xr = XmlReader.Create(entry.Open(), SheetSettings(async: false))) { var lastRow = 0; var values = new List(); @@ -460,11 +491,7 @@ private IEnumerable ReadRowsCore(ZipArchiveEntry entry) nextColumn = column + 1; var value = ReadCell(xr); // Leaves the reader on or on an empty . xr.Read(); - if (value == null) continue; - // ColumnOf caps the index at 16384, so padding is bounded even for hostile files. - if (column - 1 < values.Count) { values[column - 1] = value; continue; } // Out-of-order cell. - while (values.Count < column - 1) values.Add(null); - values.Add(value); + if (value != null) PlaceCell(values, column, value); continue; } xr.Read(); @@ -475,6 +502,17 @@ private IEnumerable ReadRowsCore(ZipArchiveEntry entry) } } + /// + /// Stores a cell value at its 1-based column in the row buffer, padding skipped columns with null. + /// ColumnOf caps the index at 16384, so the padding is bounded even for hostile files. + /// + private static void PlaceCell(List values, int column, object value) + { + if (column - 1 < values.Count) { values[column - 1] = value; return; } // Out-of-order cell. + while (values.Count < column - 1) values.Add(null); + values.Add(value); + } + /// Reads one c element (positioned on its start) and returns its typed value. private object? ReadCell(XmlReader xr) { @@ -509,6 +547,16 @@ private IEnumerable ReadRowsCore(ZipArchiveEntry entry) xr.Read(); } + return CellValue(type, styleIndex, v, inline); + } + + /// + /// Turns the raw parts of a c element (its t and s attributes, the v text and + /// any inline string) into the typed value the row exposes. Shared by the sync and async parsers so the + /// two can never disagree on how a cell reads. + /// + private object? CellValue(string? type, int styleIndex, string? v, string? inline) + { switch (type) { case "s": @@ -526,7 +574,8 @@ private IEnumerable ReadRowsCore(ZipArchiveEntry entry) return v != null && DateTime.TryParse(v, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var iso) ? iso : (object?)v; default: if (v == null || !double.TryParse(v, NumberStyles.Float, CultureInfo.InvariantCulture, out var number)) return v; - var kind = styleIndex < _styleKinds.Count ? _styleKinds[styleIndex] : CellKind.Number; + // A damaged file can carry s="-1"; treat any index outside the style table as a plain number. + var kind = styleIndex >= 0 && styleIndex < _styleKinds.Count ? _styleKinds[styleIndex] : CellKind.Number; if (kind == CellKind.Date) { if (_date1904) number += 1462; // 1904 epoch: shift to the 1900 serial Excel's OADate expects. diff --git a/src/SheetStream/RowWriter.cs b/src/SheetStream/RowWriter.cs index 6f12344..a65cdb2 100644 --- a/src/SheetStream/RowWriter.cs +++ b/src/SheetStream/RowWriter.cs @@ -29,6 +29,10 @@ public sealed class RowWriter // Computed once per row so the common path pays a single bool test per cell. private bool _observe; + // True when the sheet caches formula results and therefore wants every value recorded. Read once per + // row, so with the feature off (the default) each cell pays one false branch and nothing else. + private bool _cache; + // Row style combined with a date / date-time / duration format, derived once per row on first use. private CellStyle? _rowDateStyle; private CellStyle? _rowDateTimeStyle; @@ -70,6 +74,7 @@ internal void Begin(int rowIndex, RowOptions? options, IReadOnlyList values) } /// - /// Writes a formula cell, e.g. "SUM(C2:C101)". A leading = is removed. No cached result - /// is stored; the workbook asks Excel to recalculate on open, so the value appears when the file is - /// opened in Excel. Use an overload with a cached value when the file will also be read by tools that - /// do not calculate (viewers, ExcelReader, other libraries). + /// Writes a formula cell, e.g. "SUM(C2:C101)". A leading = is removed. By default no cached + /// result is stored; the workbook asks Excel to recalculate on open, so the value appears when the file + /// is opened in Excel. With enabled, simple aggregates over + /// cells already written get their result stored too, so tools that do not calculate (viewers, + /// ExcelReader, other libraries) show it. Pass the value yourself with the other overloads when you + /// already know it. /// public RowWriter WriteFormula(string formula, CellStyle? style = null) { var text = PrepareFormula(formula); + if (_cache && _sheet.TryEvaluateFormula(text, RowIndex, NextColumn, out var value)) + return WriteFormulaWithNumber(text, value, style); BeginCell(style, null); WriteF(text); + // A later SUM over this cell cannot know its value, so it must not get one either. + if (_cache) _sheet.RecordUnknown(RowIndex, NextColumn); EndCell(null); return this; } /// Writes a formula with a precomputed numeric result that non-calculating readers will see. public RowWriter WriteFormula(string formula, double cachedValue, CellStyle? style = null) + => WriteFormulaWithNumber(PrepareFormula(formula), cachedValue, style); + + /// Shared by the explicit cached-value overload and the evaluator path; is already prepared. + private RowWriter WriteFormulaWithNumber(string text, double cachedValue, CellStyle? style) { - var text = PrepareFormula(formula); BeginCell(style, null); WriteF(text); _w.WriteStartElement("v"); _w.WriteValue(cachedValue); _w.WriteEndElement(); + if (_cache) _sheet.RecordNumber(RowIndex, NextColumn, cachedValue); EndCell(_observe ? NumberText(cachedValue) : null); return this; } @@ -413,6 +438,7 @@ public RowWriter WriteFormula(string formula, string cachedValue, CellStyle? sty BeginCell(style, "str"); WriteF(text); _w.WriteElementString("v", XmlTextUtil.Sanitize(cachedValue)); + if (_cache) _sheet.RecordText(RowIndex, NextColumn); EndCell(cachedValue); return this; } @@ -424,6 +450,7 @@ public RowWriter WriteFormula(string formula, bool cachedValue, CellStyle? style BeginCell(style, "b"); WriteF(text); _w.WriteElementString("v", cachedValue ? "1" : "0"); + if (_cache) _sheet.RecordText(RowIndex, NextColumn); EndCell(cachedValue ? "TRUE" : "FALSE"); return this; } @@ -455,7 +482,8 @@ internal RowWriter WriteErrorValue(string errorCode, CellStyle? style) /// Writes a cell loaded from an edited workbook exactly as it was: the type attribute, any attributes the /// library does not model (cm, vm, ph) and the original inner XML (formula, cached /// value, rich inline text). Only the style is taken from , so a style change - /// does not disturb the content. + /// does not disturb the content. The caller records the cell's value for formula caching, since only it + /// knows the typed value behind the raw XML. /// internal void WriteRawCell(string? type, IReadOnlyList? attributes, string innerXml, CellStyle? style) { @@ -473,6 +501,7 @@ private RowWriter WriteError(string errorCode, CellStyle? style) _w.WriteStartElement("v"); _w.WriteString(errorCode); _w.WriteEndElement(); + if (_cache) _sheet.RecordText(RowIndex, NextColumn); EndCell(errorCode); return this; } diff --git a/src/SheetStream/Styles/StyleSheet.cs b/src/SheetStream/Styles/StyleSheet.cs index addce25..5975b13 100644 --- a/src/SheetStream/Styles/StyleSheet.cs +++ b/src/SheetStream/Styles/StyleSheet.cs @@ -267,7 +267,11 @@ private int RegisterBorder(BorderSpec border) return id; } - private int RegisterNumberFormat(string? formatCode) + /// + /// Returns the numFmtId for a format code, registering a custom format when it is not built in. + /// Pivot data fields reference number formats by id, so they share this registry with cell styles. + /// + internal int RegisterNumberFormat(string? formatCode) { if (string.IsNullOrEmpty(formatCode)) return 0; if (BuiltInNumberFormats.TryGetValue(formatCode!, out var builtIn)) return builtIn; diff --git a/src/SheetStream/Workbook.cs b/src/SheetStream/Workbook.cs index ad51bca..3995155 100644 --- a/src/SheetStream/Workbook.cs +++ b/src/SheetStream/Workbook.cs @@ -12,6 +12,7 @@ namespace SheetStream public sealed class Workbook { private readonly HashSet _tableNames = new HashSet(StringComparer.OrdinalIgnoreCase); + private readonly HashSet _pivotTableNames = new HashSet(StringComparer.OrdinalIgnoreCase); internal Workbook(ExcelPackage package) { @@ -72,6 +73,14 @@ internal void AttachTemplate(TemplateSource template) internal int ChartCount { get; set; } internal int CommentsCount { get; set; } internal int ImageCount { get; set; } + internal int PivotTableCount { get; set; } + + /// + /// Part numbers of the pivot caches written so far. Each pivot table owns one cache, and the part + /// number doubles as the cacheId that links workbook.xml to the pivot table definition. + /// The workbook part is written last, so the list is complete by then. + /// + internal List PivotCaches { get; } = new List(); /// /// Protects the workbook structure so sheets cannot be added, deleted, renamed, moved, hidden or @@ -93,5 +102,13 @@ internal void ReserveTableName(string name) if (string.Equals(dn.Name, name, StringComparison.OrdinalIgnoreCase)) throw new ArgumentException($"'{name}' is already used as a defined name.", nameof(name)); } + + /// Reserves a pivot table name; Excel numbers them workbook-wide, so duplicates are refused (case-insensitive). + internal void ReservePivotTableName(string name) + { + if (string.IsNullOrWhiteSpace(name)) throw new ArgumentException("Pivot table name cannot be empty.", nameof(name)); + if (name.Length > 255) throw new ArgumentException($"Pivot table name '{name}' is longer than 255 characters.", nameof(name)); + if (!_pivotTableNames.Add(name)) throw new ArgumentException($"A pivot table named '{name}' already exists in this workbook.", nameof(name)); + } } } diff --git a/src/SheetStream/Worksheet.Cells.cs b/src/SheetStream/Worksheet.Cells.cs index 21cc63f..ab9c9de 100644 --- a/src/SheetStream/Worksheet.Cells.cs +++ b/src/SheetStream/Worksheet.Cells.cs @@ -147,7 +147,11 @@ private void FlushBuffer() { writer.Skip(cellPair.Key - writer.NextColumn); var cell = cellPair.Value; - if (cell.RawInner != null) writer.WriteRawCell(cell.RawType, cell.RawAttributes, cell.RawInner, cell.Style); + if (cell.RawInner != null) + { + writer.WriteRawCell(cell.RawType, cell.RawAttributes, cell.RawInner, cell.Style); + RecordLoadedCell(rowPair.Key, cellPair.Key, cell); + } else if (cell.Formula != null) WriteBufferedFormula(writer, cell); else if (cell.Value == null && cell.Style != null) writer.WriteEmpty(cell.Style); else WriteBufferedValue(writer, cell); diff --git a/src/SheetStream/Worksheet.Features.cs b/src/SheetStream/Worksheet.Features.cs index 2a0643f..662d3a3 100644 --- a/src/SheetStream/Worksheet.Features.cs +++ b/src/SheetStream/Worksheet.Features.cs @@ -14,6 +14,7 @@ public sealed partial class Worksheet private readonly List _pictures = new List(); private readonly List _charts = new List(); private readonly List _comments = new List(); + private readonly List _pivotTables = new List(); private SheetProtection? _protection; // AutoFit state: widest content seen per column, in Calibri-11 character units. @@ -287,6 +288,84 @@ public ExcelChart AddChart(ChartType type, string placement) return chart; } + /// + /// Adds a pivot table to this sheet with its top-left cell at (, ), + /// summarising (header row included) on sheet . + /// Excel computes the pivot when the file opens; see for the design and its limits. + /// + /// Pivot table name, unique within the workbook (e.g. "SalesByRegion"). + /// Sheet holding the data. It must already exist in this workbook (it may be this sheet). + /// A1 range of the data including its header row, e.g. "A1:F500". + /// 1-based row of the pivot's top-left cell on this sheet. + /// 1-based column of the pivot's top-left cell on this sheet. + /// + /// The header texts of the source columns, in order. The source sheet has already been streamed out, so + /// SheetStream cannot read them back; returns them for data written with + /// a LoadFrom* call, and the overload takes + /// them from the table. + /// + /// + /// + /// var loaded = data.LoadFromCollection(orders); + /// var pivot = summary.AddPivotTable("ByRegion", "Data", loaded.Range, 1, 1, loaded.Headers); + /// pivot.AddRowField("Region").AddDataField("Amount"); + /// + /// + public PivotTable AddPivotTable(string name, string sourceSheetName, string sourceRange, int row, int column, IReadOnlyList headers) + { + if (headers == null) throw new ArgumentNullException(nameof(headers)); + if (string.IsNullOrWhiteSpace(sourceSheetName)) throw new ArgumentException("Source sheet name cannot be empty.", nameof(sourceSheetName)); + var range = RangeReference.Parse(sourceRange); + if (range.RowCount < 2) throw new ArgumentException($"Pivot source {sourceRange} needs a header row and at least one data row.", nameof(sourceRange)); + if (headers.Count != range.ColumnCount) + throw new ArgumentException($"Pivot source {sourceRange} spans {range.ColumnCount} columns but {headers.Count} headers were given.", nameof(headers)); + var source = FindSheet(sourceSheetName) ?? throw new ArgumentException($"This workbook has no sheet named '{sourceSheetName}'. Write the source sheet before adding the pivot table.", nameof(sourceSheetName)); + return AddPivotTableCore(name, source.Name, range, null, headers, row, column); + } + + /// + /// Adds a pivot table whose source is an Excel table written earlier (or on this sheet). The field names + /// come from the table's columns, and Excel re-reads the whole table on every refresh, so the pivot follows + /// the table as it grows. + /// + /// Pivot table name, unique within the workbook. + /// The table to summarise, from or . + /// 1-based row of the pivot's top-left cell on this sheet. + /// 1-based column of the pivot's top-left cell on this sheet. + public PivotTable AddPivotTable(string name, ExcelTable source, int row, int column) + { + if (source == null) throw new ArgumentNullException(nameof(source)); + if (!ReferenceEquals(source.Sheet.Workbook, Workbook)) + throw new ArgumentException($"Table '{source.Name}' belongs to another workbook.", nameof(source)); + var headers = source.ColumnNames + ?? throw new InvalidOperationException($"The header texts of table '{source.Name}' are not known yet. Write its header row (or pass column names to AddTable) before adding the pivot table."); + return AddPivotTableCore(name, source.Sheet.Name, source.Range, source.Name, headers, row, column); + } + + private PivotTable AddPivotTableCore(string name, string sourceSheet, RangeReference sourceRange, string? sourceTable, IReadOnlyList headers, int row, int column) + { + EnsureNotSealed(); + if (_templateSource != null) + throw new NotSupportedException($"Sheet '{Name}' fills a template or an edited workbook; pivot tables can only be added to sheets written from scratch."); + ValidateRow(row, nameof(row)); + CellReference.ColumnLetters(column); + // Construct first: the constructor validates the headers, and a failed call must not leave the + // name reserved or a part number consumed. + var pivot = new PivotTable(this, Workbook.PivotTableCount + 1, name, sourceSheet, sourceRange, sourceTable, headers, row, column); + Workbook.ReservePivotTableName(name); + Workbook.PivotTableCount++; + _pivotTables.Add(pivot); + return pivot; + } + + /// Finds a worksheet of this workbook by name (case-insensitive, like Excel). + private Worksheet? FindSheet(string name) + { + foreach (var s in Workbook.Worksheets) + if (string.Equals(s.Name, name, StringComparison.OrdinalIgnoreCase)) return s; + return null; + } + /// /// Attaches a comment (Excel note) to a cell. Notes appear when the mouse hovers over the cell's red /// corner mark. diff --git a/src/SheetStream/Worksheet.FormulaCache.cs b/src/SheetStream/Worksheet.FormulaCache.cs new file mode 100644 index 0000000..02cdd58 --- /dev/null +++ b/src/SheetStream/Worksheet.FormulaCache.cs @@ -0,0 +1,108 @@ +using System; +using SheetStream.Internal; + +namespace SheetStream +{ + public sealed partial class Worksheet + { + // Values written so far, kept only while CacheFormulaResults is on. Null means the feature is off and + // the writer's per-cell hook is a single false branch. + private FormulaValueStore? _formulaValues; + + /// + /// Makes also write the formula's result + /// as the cell's cached value whenever the library can compute it, so viewers that do not calculate + /// (file previews, Quick Look, many mobile and web viewers, ) show a + /// number instead of a blank. Must be enabled before the first row is written. + /// + /// + /// + /// This is not a formula engine. Supported: numbers, references to cells of this sheet that were already + /// written (rows above, or cells to the left on the same row), rectangular ranges, + - * /, unary + /// minus, a % suffix, parentheses, and SUM, AVERAGE, COUNT, COUNTA, + /// MIN, MAX, ROUND and ABS. A formula that uses anything else, references a + /// row not yet written or another sheet, or would produce an error (such as a division by zero) is + /// written exactly as before: without a cached value, and Excel computes it on open. A wrong value is + /// never written. A formula that does get a value counts as a numeric cell for later formulas, so a + /// grand total over subtotals works. + /// + /// + /// Cost: the sheet remembers every value written, about 8 bytes per cell of every column that holds + /// numbers, which is why this is opt-in. Values written through WriteCell, LoadFrom* and + /// Cells[...] are all seen. Rows copied from a template (Worksheets.Open) are not: a + /// formula referencing a template row gets no cached value. + /// + /// + /// + /// + /// sheet.CacheFormulaResults(); + /// for (var i = 0; i < 100; i++) sheet.Row().WriteCell(i).WriteCell(i * 1.5); + /// sheet.Row().WriteCell("Total").WriteFormula("SUM(B1:B100)"); // Written with its value, 7425. + /// + /// + public Worksheet CacheFormulaResults(bool enabled = true) + { + EnsureNotSealed(); + if (!enabled) + { + _formulaValues = null; + return this; + } + if (_formulaValues != null) return this; + // Enabling after some rows have gone by would leave those rows unseen, and a SUM over them would be + // wrong rather than missing; refusing is the only honest option. + if (CurrentRow > 0) + throw new InvalidOperationException($"CacheFormulaResults must be enabled on sheet '{Name}' before its first row is written, because the values of rows already streamed are gone."); + _formulaValues = new FormulaValueStore(); + return this; + } + + /// True when formula results are being cached; read by the row writer once per row. + internal bool IsCachingFormulaResults => _formulaValues != null; + + /// Records a numeric cell (numbers, dates and durations as serials, cached formula results). + internal void RecordNumber(int row, int column, double value) => _formulaValues?.SetNumber(row, column, value); + + /// Records a cell that holds non-numeric content (text, boolean, error, text or boolean formula result). + internal void RecordText(int row, int column) => _formulaValues?.SetText(row, column); + + /// Records a cell whose value is not known here (a formula written without a cached result). + internal void RecordUnknown(int row, int column) => _formulaValues?.SetUnknown(row, column); + + /// Records a copied template row whose cells were never seen. + private void RecordUnknownRow(int row) => _formulaValues?.MarkRowUnknown(row); + + /// + /// Records a cell flushed from the random-access buffer with its original XML (edit mode), from the typed + /// value the loader produced. A formula without a cached result stays unknown. + /// + private void RecordLoadedCell(int row, int column, BufferedCell cell) + { + if (_formulaValues == null) return; + switch (cell.Value) + { + case null: + if (cell.Formula != null) _formulaValues.SetUnknown(row, column); + break; + case double d: _formulaValues.SetNumber(row, column, d); break; + case DateTime date: + // Mirror what the writer would store: a 1904-system workbook counts serial days from a later epoch. + _formulaValues.SetNumber(row, column, date.ToOADate() - (_templateSource != null && _templateSource.Date1904 ? 1462 : 0)); + break; + case TimeSpan span: _formulaValues.SetNumber(row, column, span.TotalDays); break; + default: _formulaValues.SetText(row, column); break; + } + } + + /// Evaluates a formula about to be written at the given cell; false when no value can be cached. + internal bool TryEvaluateFormula(string formula, int row, int column, out double value) + { + if (_formulaValues == null) + { + value = 0; + return false; + } + return FormulaEvaluator.TryEvaluate(formula, _formulaValues, row, column, Name, out value); + } + } +} diff --git a/src/SheetStream/Worksheet.Load.cs b/src/SheetStream/Worksheet.Load.cs index f7f4873..073f508 100644 --- a/src/SheetStream/Worksheet.Load.cs +++ b/src/SheetStream/Worksheet.Load.cs @@ -177,19 +177,20 @@ private LoadResult LoadCore(string[] headers, CellStyle?[] memberStyles, LoadOpt } var lastRow = CurrentRow < firstRow ? firstRow : CurrentRow; + ExcelTable? table = null; if (options.TableName != null) { // A table needs at least one data row; Excel rejects a header-only table. if (rows == 0) Row().Skip(options.StartColumn - 1).WriteEmpty(); lastRow = CurrentRow; - var table = AddTable(options.TableName, firstRow, options.StartColumn, lastRow, lastColumn, headers); + table = AddTable(options.TableName, firstRow, options.StartColumn, lastRow, lastColumn, headers); table.StyleName = options.TableStyle; } else if (options.AutoFilter && options.PrintHeaders && headers.Length > 0) { AutoFilter(firstRow, options.StartColumn, lastColumn); } - return new LoadResult(firstRow, lastRow, options.StartColumn, lastColumn, rows); + return new LoadResult(firstRow, lastRow, options.StartColumn, lastColumn, rows, (string[])headers.Clone(), table); } /// diff --git a/src/SheetStream/Worksheet.Template.cs b/src/SheetStream/Worksheet.Template.cs index 651bcb7..6f3d303 100644 --- a/src/SheetStream/Worksheet.Template.cs +++ b/src/SheetStream/Worksheet.Template.cs @@ -182,7 +182,12 @@ private void MergeTemplateRowsBefore(int index) { var r = _templateReader!; _templateLastRow = _templateRowAhead.Value; - if (_templateRowAhead.Value < index) XmlCopy.CopySubtree(r, _part); + if (_templateRowAhead.Value < index) + { + // Copied as XML without looking at its cells, so formulas must not treat it as known. + RecordUnknownRow(_templateRowAhead.Value); + XmlCopy.CopySubtree(r, _part); + } else if (SkipTemplateRow(r)) ReplacedTemplateRows++; ParkOnNextTemplateRow(); } diff --git a/src/SheetStream/Worksheet.cs b/src/SheetStream/Worksheet.cs index ca7f5a8..16b0a7f 100644 --- a/src/SheetStream/Worksheet.cs +++ b/src/SheetStream/Worksheet.cs @@ -39,7 +39,7 @@ public sealed class RowOptions /// sheetFormatPr, cols, sheetData start); the rows; and a trailer (protection, /// filters, merges, conditional formats, validations, hyperlinks, print settings, drawings, tables) /// written when the sheet is sealed by adding the next sheet or saving the package. Parts that live - /// outside the sheet (tables, drawings, charts, comments, the sheet's relationships) are written right + /// outside the sheet (tables, drawings, charts, comments, pivot tables, the sheet's relationships) are written right /// after it. Not thread-safe. /// public sealed partial class Worksheet @@ -607,6 +607,13 @@ private void WriteSubParts(RelationshipSet rels) } if (_drawingNumber > 0) DrawingWriter.Write(package, Workbook, _drawingNumber, _pictures, _charts); if (_commentsNumber > 0) CommentsWriter.Write(package, _commentsNumber, Index, _comments, out _, out _); + // A pivot table is referenced only from the sheet's .rels, never from the sheet XML, so its + // relationship can be allocated here, after the sheet part is closed. + foreach (var p in _pivotTables) + { + rels.Add(OpenXmlNamespaces.RelPivotTable, "../pivotTables/pivotTable" + p.Number.ToString(CultureInfo.InvariantCulture) + ".xml"); + PivotTableWriter.Write(package, Workbook, p); + } if (rels.Count > 0) using (var w = package.OpenPart("xl/worksheets/_rels/sheet" + Index.ToString(CultureInfo.InvariantCulture) + ".xml.rels")) rels.WriteTo(w); diff --git a/tests/SheetStream.Tests/AsyncReaderTests.cs b/tests/SheetStream.Tests/AsyncReaderTests.cs new file mode 100644 index 0000000..9e04e08 --- /dev/null +++ b/tests/SheetStream.Tests/AsyncReaderTests.cs @@ -0,0 +1,371 @@ +using System.Data; +using System.Runtime.CompilerServices; +using SheetStream.Reading; +using SheetStream.Styles; + +namespace SheetStream.Tests; + +/// +/// The asynchronous reader (OpenAsync, ReadRowsAsync, ReadAsync<T>, ReadDataTableAsync). Every workbook is +/// written with SheetStream and the async results are compared with the synchronous API on the same bytes, so +/// the two parsers are proven to agree cell for cell. +/// +public class AsyncReaderTests +{ + public sealed class Product + { + public int Id { get; set; } + public string Name { get; set; } = ""; + public double Price { get; set; } + public bool InStock { get; set; } + public DateTime? Shipped { get; set; } + } + + /// A workbook with every value kind the reader types, gaps, blank rows and a second sheet. + private static byte[] Workbook(int dataRows = 25) => ExcelPackage.CreateBytes(wb => + { + var s = wb.Worksheets.Add("Products"); + s.Row().WriteCells("Id", "Name", "Price", "InStock", "Shipped"); + for (var i = 1; i <= dataRows; i++) + { + var r = s.Row().WriteCell(i).WriteCell("Product " + i + (i % 7 == 0 ? " & co " : "")).WriteCell(i * 1.25).WriteCell(i % 2 == 0); + if (i % 3 == 0) r.WriteEmpty(); else r.WriteCell(new DateTime(2026, 1, 1).AddDays(i)); + if (i % 5 == 0) s.Row(); // A blank row: ReadRows yields it empty, Read skips it. + } + s.Row().Skip(5).WriteCell(TimeSpan.FromHours(1.5)).WriteCell("far right"); // Sparse row (F, G) outside the header: Read skips it. + wb.Worksheets.Add("Second").Row().WriteCells("a", 1.0); + }); + + /// Hides seekability, the way an HTTP request body or a network stream does. + private sealed class ForwardOnlyStream(Stream inner) : Stream + { + public bool Disposed { get; private set; } + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => throw new NotSupportedException(); + public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); } + public override void Flush() { } + public override int Read(byte[] buffer, int offset, int count) => inner.Read(buffer, offset, count); + public override async ValueTask ReadAsync(Memory buffer, CancellationToken ct = default) + { + await Task.Yield(); // Behave like a real network stream: complete asynchronously. + return await inner.ReadAsync(buffer.Slice(0, Math.Min(buffer.Length, 1000)), ct); // Small chunks exercise the loop. + } + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + protected override void Dispose(bool disposing) { Disposed = true; base.Dispose(disposing); } + } + + /// A hand-written mapper, as a user without the source generator would write one. + private sealed class ProductMapper : ISheetRowMapper + { + public IReadOnlyList Headers { get; } = new[] { "Id", "Name", "Price" }; + public IReadOnlyList NumberFormats { get; } = new string?[3]; + public bool CanRead => true; + public Type MemberType(int column) => column switch { 0 => typeof(int), 1 => typeof(string), _ => typeof(double) }; + public bool CanSet(int column) => true; + public int FindColumn(string header) + { + for (var i = 0; i < Headers.Count; i++) + if (string.Equals(Headers[i], header, StringComparison.OrdinalIgnoreCase)) return i; + return -1; + } + public void WriteRow(Product item, RowWriter row, CellStyle?[] styles) => row.WriteCell(item.Id).WriteCell(item.Name).WriteCell(item.Price); + public object? GetValue(Product item, int column) => column switch { 0 => item.Id, 1 => item.Name, _ => item.Price }; + public Product Create() => new(); + public void SetValue(ref Product item, int column, object? value) + { + switch (column) + { + case 0: item.Id = (int)value!; break; + case 1: item.Name = (string)value!; break; + case 2: item.Price = (double)value!; break; + } + } + } + + private static async Task> CollectAsync(IAsyncEnumerable rows) + { + var list = new List(); + await foreach (var row in rows) list.Add(row); + return list; + } + + private static void AssertSameRows(IReadOnlyList expected, IReadOnlyList actual) + { + Assert.Equal(expected.Count, actual.Count); + for (var i = 0; i < expected.Count; i++) + { + Assert.Equal(expected[i].Index, actual[i].Index); + Assert.Equal(expected[i].Values, actual[i].Values); // Typed compare: double vs double, DateTime vs DateTime... + } + } + + [Fact] + public async Task ReadRowsAsync_MatchesSyncRowByRow() + { + var bytes = Workbook(); + using var reader = ExcelReader.Open(new MemoryStream(bytes)); + var sync = reader.ReadRows("Products").ToList(); + var async = await CollectAsync(reader.ReadRowsAsync("Products")); + + AssertSameRows(sync, async); + // Sanity: the data really covers every value kind the reader produces. + var kinds = async.SelectMany(r => r.Values).Where(v => v != null).Select(v => v!.GetType()).Distinct().ToHashSet(); + Assert.Superset(new HashSet { typeof(string), typeof(double), typeof(bool), typeof(DateTime), typeof(TimeSpan) }, kinds); + Assert.Contains(async, r => r.Values.Count == 0); // Blank rows survive as empty rows. + Assert.Equal("Product 7 & co ", async.First(r => r.Index == 9).GetString(1)); // Row 7 is the blank row after product 5. + } + + [Fact] + public async Task ReadRowsAsync_ByIndex_AndCaseInsensitiveName() + { + using var reader = ExcelReader.Open(new MemoryStream(Workbook(3))); + var byName = await CollectAsync(reader.ReadRowsAsync("PRODUCTS")); + var byIndex = await CollectAsync(reader.ReadRowsAsync(0)); + AssertSameRows(byName, byIndex); + + var second = await CollectAsync(reader.ReadRowsAsync(1)); + Assert.Single(second); + Assert.Equal(new object?[] { "a", 1.0 }, second[0].Values); + } + + [Fact] + public void ReadRowsAsync_FailsEagerlyForUnknownSheet() + { + using var reader = ExcelReader.Open(new MemoryStream(Workbook(1))); + // Validation happens on the call, not on the first MoveNextAsync, like the sync API. + Assert.Throws(() => reader.ReadRowsAsync("Nope")); + Assert.Throws(() => reader.ReadRowsAsync(5)); + Assert.Throws(() => reader.ReadAsync("Nope")); + Assert.Throws(() => reader.ReadAsync("Nope", new ProductMapper())); + } + + [Fact] + public async Task ReadAsync_Reflection_MatchesSync() + { + using var reader = ExcelReader.Open(new MemoryStream(Workbook())); + var sync = reader.Read("Products").ToList(); + var async = new List(); + await foreach (var p in reader.ReadAsync("Products")) async.Add(p); + + Assert.Equal(25, async.Count); // Blank rows and the sparse row (no bound column) are skipped. + Assert.Equal(sync.Select(p => (p.Id, p.Name, p.Price, p.InStock, p.Shipped)), async.Select(p => (p.Id, p.Name, p.Price, p.InStock, p.Shipped))); + Assert.Equal(new DateTime(2026, 1, 2), async[0].Shipped); + Assert.Null(async[2].Shipped); // Row 3 was written empty. + } + + [Fact] + public async Task ReadAsync_WithHandWrittenMapper() + { + using var reader = ExcelReader.Open(new MemoryStream(Workbook(10))); + var items = new List(); + await foreach (var p in reader.ReadAsync("Products", new ProductMapper())) items.Add(p); + + Assert.Equal(10, items.Count); + Assert.Equal(Enumerable.Range(1, 10), items.Select(p => p.Id)); + Assert.Equal("Product 7 & co ", items[6].Name); + Assert.Equal(12.5, items[9].Price); + Assert.All(items, p => Assert.False(p.InStock)); // Not a mapper column: left at its default. + + var sync = reader.Read("Products", new ProductMapper()).Select(p => (p.Id, p.Name, p.Price)); + Assert.Equal(sync, items.Select(p => (p.Id, p.Name, p.Price))); + } + + [Fact] + public async Task ReadAsync_PrefersRegisteredMapper() + { + SheetMappers.Register(new ProductMapper()); + try + { + using var reader = ExcelReader.Open(new MemoryStream(Workbook(4))); + var items = new List(); + await foreach (var p in reader.ReadAsync("Products")) items.Add(p); + // The registered mapper does not bind InStock, so the reflection path was not taken. + Assert.Equal(4, items.Count); + Assert.All(items, p => Assert.False(p.InStock)); + } + finally + { + SheetMappers.Unregister(); + } + } + + [Fact] + public async Task ReadDataTableAsync_MatchesSync() + { + using var reader = ExcelReader.Open(new MemoryStream(Workbook())); + var sync = reader.ReadDataTable("Products"); + var async = await reader.ReadDataTableAsync("Products"); + + Assert.Equal(sync.Columns.Cast().Select(c => c.ColumnName), async.Columns.Cast().Select(c => c.ColumnName)); + Assert.Equal(sync.Rows.Count, async.Rows.Count); + for (var i = 0; i < sync.Rows.Count; i++) Assert.Equal(sync.Rows[i].ItemArray, async.Rows[i].ItemArray); + Assert.Equal(7, async.Columns.Count); // Header has 5; the sparse row widened the table to G. + + var noHeader = await reader.ReadDataTableAsync("Products", firstRowIsHeader: false); + Assert.Equal("A", noHeader.Columns[0].ColumnName); + Assert.Equal(sync.Rows.Count + 1, noHeader.Rows.Count); + } + + [Fact] + public async Task OpenAsync_NonSeekableStream_IsBufferedAndClosed() + { + var bytes = Workbook(); + var source = new ForwardOnlyStream(new MemoryStream(bytes)); + using var reader = await ExcelReader.OpenAsync(source); + + Assert.True(source.Disposed); // leaveOpen defaults to false: the upload stream is released once copied. + Assert.Equal(new[] { "Products", "Second" }, reader.SheetNames); + using var expected = ExcelReader.Open(new MemoryStream(bytes)); + AssertSameRows(expected.ReadRows("Products").ToList(), await CollectAsync(reader.ReadRowsAsync("Products"))); + } + + [Fact] + public async Task OpenAsync_LeaveOpen_KeepsTheSourceAlive() + { + var forwardOnly = new ForwardOnlyStream(new MemoryStream(Workbook(2))); + using (var reader = await ExcelReader.OpenAsync(forwardOnly, leaveOpen: true)) + Assert.Equal(2, reader.SheetNames.Count); + Assert.False(forwardOnly.Disposed); + + var seekable = new MemoryStream(Workbook(2)); + using (var reader = await ExcelReader.OpenAsync(seekable, leaveOpen: true)) + Assert.Equal(4, (await CollectAsync(reader.ReadRowsAsync(0))).Count); // Header, two products, the sparse row. + Assert.True(seekable.CanRead); // Still open after the reader is disposed. + } + + [Fact] + public async Task OpenAsync_NonSeekable_SpillsToTempFileAboveTheLimit() + { + var bytes = Workbook(200); + // The public limit is 64 MB; the internal entry point lets the test force the spill with a small one. + using var buffered = await ExcelReader.BufferAsync(new ForwardOnlyStream(new MemoryStream(bytes)), memoryLimit: 4096, CancellationToken.None); + + var file = Assert.IsType(buffered); + Assert.True(File.Exists(file.Name)); + Assert.Equal(0, buffered.Position); + Assert.Equal(bytes.Length, buffered.Length); + var copy = new byte[bytes.Length]; + Assert.Equal(bytes.Length, await buffered.ReadAtLeastAsync(copy, bytes.Length)); + Assert.Equal(bytes, copy); + buffered.Position = 0; + + using (var reader = ExcelReader.Open(buffered, leaveOpen: true)) + Assert.Equal(200 + 1 + 40 + 1, (await CollectAsync(reader.ReadRowsAsync("Products"))).Count); + buffered.Dispose(); + Assert.False(File.Exists(file.Name)); // DeleteOnClose: nothing is left behind. + + using var small = await ExcelReader.BufferAsync(new ForwardOnlyStream(new MemoryStream(bytes)), memoryLimit: long.MaxValue, CancellationToken.None); + Assert.IsType(small); + Assert.Equal(bytes.Length, small.Length); + } + + [Fact] + public async Task OpenAsync_Path_ReadsTheFile() + { + var path = Path.Combine(Path.GetTempPath(), "sheetstream-async-" + Guid.NewGuid().ToString("N") + ".xlsx"); + await File.WriteAllBytesAsync(path, Workbook(5)); + try + { + using var reader = await ExcelReader.OpenAsync(path); + var rows = await CollectAsync(reader.ReadRowsAsync("Products")); + Assert.Equal(5 + 1 + 1 + 1, rows.Count); // header, 5 products, one blank row (after 5), sparse row. + Assert.Equal(1.0, rows[1][0]); + } + finally + { + File.Delete(path); + } + + await Assert.ThrowsAsync(() => ExcelReader.OpenAsync(path)); // Reported through the task. + await Assert.ThrowsAsync(async () => await ExcelReader.OpenAsync(default(string)!)); + } + + [Fact] + public async Task OpenAsync_HonoursCancellationWhileBuffering() + { + using var cts = new CancellationTokenSource(); + cts.Cancel(); + await Assert.ThrowsAnyAsync(() => ExcelReader.OpenAsync(new ForwardOnlyStream(new MemoryStream(Workbook(1))), cancellationToken: cts.Token)); + await Assert.ThrowsAnyAsync(() => ExcelReader.OpenAsync("whatever.xlsx", cts.Token)); + } + + [Fact] + public async Task ReadRowsAsync_CancellationMidSheet_ThrowsAndLeavesReaderUsable() + { + using var reader = ExcelReader.Open(new MemoryStream(Workbook(100))); + using var cts = new CancellationTokenSource(); + var seen = 0; + + var ex = await Assert.ThrowsAnyAsync(async () => + { + await foreach (var row in reader.ReadRowsAsync("Products", cts.Token)) + { + if (++seen == 10) cts.Cancel(); // The row in hand is delivered; the next MoveNextAsync throws. + } + }); + Assert.Equal(cts.Token, ex.CancellationToken); + Assert.Equal(10, seen); + + // The iterator's finally ran: the entry stream is closed and the sheet can be read again in full. + Assert.Equal(reader.ReadRows("Products").Count(), (await CollectAsync(reader.ReadRowsAsync("Products"))).Count); + reader.Dispose(); // And disposing afterwards is clean (the outer using disposes a second time, which must also be harmless). + } + + [Fact] + public async Task ReadRowsAsync_WithCancellation_PreCancelledTokenThrowsBeforeTheFirstRow() + { + using var reader = ExcelReader.Open(new MemoryStream(Workbook(3))); + using var cts = new CancellationTokenSource(); + cts.Cancel(); + + // Token supplied through WithCancellation is combined with the one passed to the method. + await Assert.ThrowsAnyAsync(async () => + { + await foreach (var _ in reader.ReadRowsAsync("Products").WithCancellation(cts.Token)) Assert.Fail("No row should be produced."); + }); + await Assert.ThrowsAnyAsync(async () => + { + await foreach (var _ in reader.ReadAsync("Products", cts.Token)) Assert.Fail("No item should be produced."); + }); + await Assert.ThrowsAnyAsync(() => reader.ReadDataTableAsync("Products", cancellationToken: cts.Token)); + } + + [Fact] + public async Task ReadRowsAsync_BreakingEarly_ReleasesTheSheet() + { + using var reader = ExcelReader.Open(new MemoryStream(Workbook(50))); + var firstTwo = new List(); + await foreach (var row in reader.ReadRowsAsync("Products")) + { + firstTwo.Add(row.Index); + if (firstTwo.Count == 2) break; + } + Assert.Equal(new[] { 1, 2 }, firstTwo); + + // Two interleaved enumerations and a full re-read all work: nothing was left half-open. + await using var a = reader.ReadRowsAsync("Products").GetAsyncEnumerator(); + await using var b = reader.ReadRowsAsync("Second").GetAsyncEnumerator(); + Assert.True(await a.MoveNextAsync()); + Assert.True(await b.MoveNextAsync()); + Assert.Equal("a", b.Current.GetString(0)); + Assert.Equal(reader.ReadRows("Products").Count(), (await CollectAsync(reader.ReadRowsAsync("Products"))).Count); + } + + [Fact] + public async Task EndToEnd_UploadRoundTrip() + { + // The scenario the API exists for: a request body arrives as a non-seekable stream and is mapped to objects. + var products = Enumerable.Range(1, 300).Select(i => new Product { Id = i, Name = "P" + i, Price = i * 0.5, InStock = i % 2 == 0, Shipped = i % 4 == 0 ? null : new DateTime(2026, 3, 1).AddHours(i) }).ToList(); + var bytes = ExcelPackage.CreateBytes(wb => wb.Worksheets.Add("Products").LoadFromCollection(products)); + + using var reader = await ExcelReader.OpenAsync(new ForwardOnlyStream(new MemoryStream(bytes))); + var back = new List(); + await foreach (var p in reader.ReadAsync("Products").WithCancellation(CancellationToken.None)) back.Add(p); + + Assert.Equal(products.Select(p => (p.Id, p.Name, p.Price, p.InStock, p.Shipped)), back.Select(p => (p.Id, p.Name, p.Price, p.InStock, p.Shipped))); + } +} diff --git a/tests/SheetStream.Tests/FormulaCacheTests.cs b/tests/SheetStream.Tests/FormulaCacheTests.cs new file mode 100644 index 0000000..4e8ccfe --- /dev/null +++ b/tests/SheetStream.Tests/FormulaCacheTests.cs @@ -0,0 +1,444 @@ +// The Open XML SDK exposes nearly every member as nullable; in test assertions a null is a failure anyway. +#pragma warning disable CS8602, CS8603, CS8604 +using System.IO.Compression; +using System.Linq; +using DocumentFormat.OpenXml; +using DocumentFormat.OpenXml.Packaging; +using DocumentFormat.OpenXml.Validation; +using SheetStream.Internal; +using SheetStream.Reading; +using X = DocumentFormat.OpenXml.Spreadsheet; + +namespace SheetStream.Tests; + +/// +/// Formula result caching: the evaluator's grammar and limits in isolation, then end-to-end files read back +/// through ExcelReader, which only ever sees the cached <v>. +/// +public class FormulaCacheTests +{ + // ----- Evaluator ------------------------------------------------------------------------------------------- + + /// + /// A small sheet already written up to row 9: B1:B5 = 10..50, C1:C5 = 1.5, 2.5, "text", (empty), 4.5, + /// D1 = "x". The formula under test is written at A10 unless a test says otherwise. + /// + private static FormulaValueStore SampleStore() + { + var store = new FormulaValueStore(); + for (var r = 1; r <= 5; r++) store.SetNumber(r, 2, r * 10); + store.SetNumber(1, 3, 1.5); + store.SetNumber(2, 3, 2.5); + store.SetText(3, 3); + store.SetNumber(5, 3, 4.5); + store.SetText(1, 4); + return store; + } + + private static double? Eval(string formula, FormulaValueStore? store = null, int row = 10, int column = 1, string sheet = "Data") + => FormulaEvaluator.TryEvaluate(formula, store ?? SampleStore(), row, column, sheet, out var v) ? v : (double?)null; + + [Theory] + [InlineData("1+2*3", 7)] + [InlineData("(1+2)*3", 9)] + [InlineData("10-4-3", 3)] // Left associative. + [InlineData("10/4", 2.5)] + [InlineData("2*-3", -6)] + [InlineData("-B1", -10)] + [InlineData("--B1", 10)] + [InlineData("50%", 0.5)] + [InlineData("B1*21%", 2.1)] + [InlineData("1E3+1", 1001)] + [InlineData(".5+.25", 0.75)] + [InlineData(" 1 + 2 ", 3)] + [InlineData("B1 * B2 / B3", 20.0 / 3)] + [InlineData("C4 + 1", 1)] // An empty cell is 0 in arithmetic. + [InlineData("$B$1+B$2+$B3", 60)] + [InlineData("b1+b2", 30)] // References are case-insensitive. + [InlineData("SUM(B1:B5)", 150)] + [InlineData("SUM($B$1:$B$5)", 150)] + [InlineData("sum(B5:B1)", 150)] // Reversed corners are normalised. + [InlineData("SUM(B1:C5)", 158.5)] // Two columns; text and empties skipped. + [InlineData("SUM(B1:B5, 10)", 160)] + [InlineData("SUM(B1, B2)", 30)] + [InlineData("SUM(B1, B2*2, 1)", 51)] + [InlineData("SUM(C1:C5)", 8.5)] + [InlineData("SUM(C3)", 0)] // A lone text cell in an aggregate is ignored, as in Excel. + [InlineData("AVERAGE(B1:B5)", 30)] + [InlineData("AVERAGE(C1:C5)", 8.5 / 3)] + [InlineData("COUNT(C1:C5)", 3)] + [InlineData("COUNT(C1:C5, 7)", 4)] + [InlineData("COUNTA(C1:C5)", 4)] + [InlineData("MIN(B1:B5)", 10)] + [InlineData("MAX(B1:B5)", 50)] + [InlineData("MIN(C4)", 0)] // MIN/MAX over nothing numeric is 0 in Excel. + [InlineData("MAX(B1:B5, 99)", 99)] + [InlineData("ROUND(B1/3, 2)", 3.33)] + [InlineData("ROUND(2.5, 0)", 3)] // Half away from zero, not banker's rounding. + [InlineData("ROUND(-2.5, 0)", -3)] + [InlineData("ROUND(1234.5, -2)", 1200)] + [InlineData("ROUND(B2, 1)", 20)] + [InlineData("ABS(-B1)", 10)] + [InlineData("ABS(B1-B2)", 10)] + [InlineData("ABS(C4)", 0)] + [InlineData("'Data'!B1 + Data!B2", 30)] + [InlineData("SUM('Data'!B1:B5)", 150)] + [InlineData("SUM('DATA'!B1:B5)", 150)] // Sheet names are case-insensitive. + [InlineData("SUM(B1:B5)*2+ABS(-1)", 301)] + [InlineData("ROUND(SUM(B1:B5)/COUNT(B1:B5), 0)", 30)] + public void Evaluator_ComputesSupportedFormulas(string formula, double expected) + { + var actual = Eval(formula); + Assert.NotNull(actual); + Assert.Equal(expected, actual.Value, 10); + } + + [Theory] + [InlineData("SUM(B1:B10)")] // Reaches the row being written. + [InlineData("B10")] // The cell itself. + [InlineData("B11")] // A row below. + [InlineData("SUM(B1:B5) + C3")] // Text in arithmetic is #VALUE!. + [InlineData("C3*2")] + [InlineData("D1")] + [InlineData("'Other'!B1")] // Another sheet's values are not available. + [InlineData("SUM(Other!B1:B5)")] + [InlineData("B1:B5")] // A bare range is not a number. + [InlineData("IF(B1>5, 1, 0)")] // Unsupported function. + [InlineData("SUMIF(B1:B5, \">1\")")] + [InlineData("B1>5")] // Comparisons. + [InlineData("B1=B2")] + [InlineData("\"a\"&\"b\"")] // Strings and concatenation. + [InlineData("2^3")] // Power is not in the subset. + [InlineData("Total")] // Defined names. + [InlineData("SUM(Lines[Amount])")] // Structured references. + [InlineData("1/0")] // #DIV/0! + [InlineData("B1/C4")] // Division by an empty cell is also #DIV/0!. + [InlineData("AVERAGE(C4)")] // AVERAGE of nothing is #DIV/0!. + [InlineData("SUM()")] + [InlineData("ROUND(1.5)")] // Wrong arity. + [InlineData("ROUND(1.5, 0, 1)")] + [InlineData("ABS(1, 2)")] + [InlineData("ABS(B1:B2)")] // A scalar function with a range. + [InlineData("(1+2")] // Malformed input. + [InlineData("1+")] + [InlineData("SUM(B1:B5")] + [InlineData("1 2")] + [InlineData("")] + [InlineData("A1B+1")] + [InlineData("1.2.3")] + [InlineData("SUM(B1:B5,)")] + public void Evaluator_RefusesWhatItCannotPromise(string formula) + => Assert.Null(Eval(formula)); + + [Fact] + public void Evaluator_SeesCellsToTheLeftOnTheSameRow_ButNotToTheRight() + { + var store = SampleStore(); + store.SetNumber(10, 2, 3); // B10 = 3, written before D10. + store.SetNumber(10, 3, 4); // C10 = 4. + Assert.Equal(12, Eval("B10*C10", store, row: 10, column: 4)); + Assert.Equal(8, Eval("SUM(B10:C10)+1", store, row: 10, column: 4)); + Assert.Null(Eval("E10", store, row: 10, column: 4)); // Not written yet. + Assert.Null(Eval("SUM(B10:E10)", store, row: 10, column: 4)); + Assert.Null(Eval("D10", store, row: 10, column: 4)); // The cell itself. + } + + [Fact] + public void Evaluator_FailsOnUnknownCellsAndOpaqueRows() + { + var store = SampleStore(); + store.SetUnknown(3, 2); // B3 holds a formula whose result was never known. + Assert.Null(Eval("SUM(B1:B5)", store)); + Assert.Null(Eval("B3+1", store)); + Assert.Equal(30, Eval("SUM(B1:B2)", store)); + + var template = SampleStore(); + template.MarkRowUnknown(4); // A row copied from a template. + Assert.Null(Eval("SUM(B1:B5)", template)); + Assert.Null(Eval("B4", template)); + Assert.Null(Eval("COUNTA(B4:B4)", template)); + Assert.Equal(60, Eval("SUM(B1:B3)", template)); + } + + [Fact] + public void ValueStore_DistinguishesZeroEmptyTextAndUnknown_AndGrows() + { + var store = new FormulaValueStore(); + store.SetNumber(1, 1, 0); + store.SetText(2, 1); + store.SetUnknown(3, 1); + store.SetNumber(100_000, 5_000, -1.25); // Far away: forces both arrays to grow. + Assert.Equal(FormulaCellState.Number, store.Get(1, 1, out var zero)); + Assert.Equal(0, zero); + Assert.Equal(FormulaCellState.Text, store.Get(2, 1, out _)); + Assert.Equal(FormulaCellState.Unknown, store.Get(3, 1, out _)); + Assert.Equal(FormulaCellState.Empty, store.Get(4, 1, out _)); + Assert.Equal(FormulaCellState.Empty, store.Get(1, 2, out _)); + Assert.Equal(FormulaCellState.Empty, store.Get(99_999, 5_000, out _)); + Assert.Equal(FormulaCellState.Number, store.Get(100_000, 5_000, out var far)); + Assert.Equal(-1.25, far); + Assert.Equal(FormulaCellState.Empty, store.Get(1, 16_384, out _)); + } + + // ----- End to end ------------------------------------------------------------------------------------------ + + private static SpreadsheetDocument OpenValidated(byte[] bytes) + { + var doc = SpreadsheetDocument.Open(new MemoryStream(bytes), false); + var errors = new OpenXmlValidator(FileFormatVersions.Office2016).Validate(doc).ToList(); + Assert.True(errors.Count == 0, string.Join(Environment.NewLine, errors.Select(e => $"{e.Path?.PartUri} {e.Path?.XPath}: {e.Description}"))); + return doc; + } + + private static string SheetXml(byte[] bytes, int sheetNumber = 1) + { + using var zip = new ZipArchive(new MemoryStream(bytes), ZipArchiveMode.Read); + using var reader = new StreamReader(zip.GetEntry($"xl/worksheets/sheet{sheetNumber}.xml").Open()); + return reader.ReadToEnd(); + } + + private static Dictionary ReadRows(byte[] bytes, string sheet) + { + using var reader = ExcelReader.Open(new MemoryStream(bytes)); + return reader.ReadRows(sheet).ToDictionary(r => r.Index); + } + + /// 100 amount rows under a header, then a block of aggregate formulas; toggles the feature. + private static byte[] BuildReport(bool cache) + { + return ExcelPackage.CreateBytes(wb => + { + var money = wb.Styles.Create(o => o.NumberFormat = "#,##0.00"); + var s = wb.Worksheets.Add("Q1 Report"); + if (cache) s.CacheFormulaResults(); + s.Row().WriteCell("#").WriteCell("Item").WriteCell("Amount"); + for (var i = 1; i <= 100; i++) s.Row().WriteCell(i).WriteCell("Item " + i).WriteCell(i * 1.25m, money); + s.Row(102).WriteCell("Total").Skip().WriteFormula("SUM(C2:C101)", money); + s.Row(103).WriteCell("Average").Skip().WriteFormula("=AVERAGE(C2:C101)"); + s.Row(104).WriteCell("Count").Skip().WriteFormula("COUNT(C2:C101)"); + s.Row(105).WriteCell("Half, rounded").Skip().WriteFormula("ROUND(C102/2, 1)"); + s.Row(106).WriteCell("Items").Skip().WriteFormula("COUNTA(B2:B101)"); + s.Row(107).WriteCell("Qualified").Skip().WriteFormula("SUM('Q1 Report'!C2:C101)"); + s.Row(108).WriteCell("Forward").Skip().WriteFormula("SUM(C2:C200)"); // Reaches past the current row. + s.Row(109).WriteCell("Unsupported").Skip().WriteFormula("IF(C102>0,\"ok\",\"no\")"); + s.Row(110).WriteCell("Elsewhere").Skip().WriteFormula("SUM(Other!C2:C101)"); + }); + } + + [Fact] + public void Streaming_WritesCachedResults_ReadableByExcelReader() + { + var bytes = BuildReport(cache: true); + using (var doc = OpenValidated(bytes)) + Assert.True(doc.WorkbookPart.Workbook.CalculationProperties.FullCalculationOnLoad.Value); // Excel still recalculates. + + var rows = ReadRows(bytes, "Q1 Report"); + Assert.Equal(6312.5, (double)rows[102][2]!, 10); // 1.25 * (1 + ... + 100) + Assert.Equal(63.125, (double)rows[103][2]!, 10); + Assert.Equal(100.0, rows[104][2]); + Assert.Equal(3156.3, (double)rows[105][2]!, 10); // 3156.25 rounded half away from zero. + Assert.Equal(100.0, rows[106][2]); + Assert.Equal(6312.5, (double)rows[107][2]!, 10); + Assert.Null(rows[108][2]); // Forward reference: no cached value. + Assert.Null(rows[109][2]); // Unsupported function: no cached value. + Assert.Null(rows[110][2]); // Other sheet: no cached value. + + var xml = SheetXml(bytes); + Assert.Contains("SUM(C2:C101)6312.5", xml); + Assert.Contains("SUM(C2:C200)", xml); + Assert.Contains("IF(C102>0,\"ok\",\"no\")", xml); + } + + [Fact] + public void Disabled_WritesNoCachedValues() + { + var bytes = BuildReport(cache: false); + var xml = SheetXml(bytes); + Assert.Contains("SUM(C2:C101)", xml); + Assert.Contains("AVERAGE(C2:C101)", xml); + Assert.DoesNotContain("", xml); + + var rows = ReadRows(bytes, "Q1 Report"); + Assert.Null(rows[102][2]); + Assert.Null(rows[103][2]); + Assert.Equal(1.25, rows[2][2]); // Plain values are of course still there. + } + + [Fact] + public void FormulaResults_FeedLaterFormulas_AndSameRowReferencesWork() + { + var bytes = ExcelPackage.CreateBytes(wb => + { + var s = wb.Worksheets.Add("Lines"); + s.CacheFormulaResults(); + s.Row().WriteCell("Qty").WriteCell("Price").WriteCell("Line total").WriteCell("Check"); + s.Row().WriteCell(2).WriteCell(9.5).WriteFormula("A2*B2").WriteFormula("C2/A2"); // 19, 9.5 + s.Row().WriteCell(3).WriteCell(4).WriteFormula("A3*B3").WriteFormula("E3"); // 12; E3 is to the right: none + s.Row().WriteCell("Subtotal").Skip().WriteFormula("SUM(C2:C3)"); // 31 + s.Row().WriteCell("Tax").Skip().WriteFormula("ROUND(C4*21%, 2)"); // 6.51 + s.Row().WriteCell("Total").Skip().WriteFormula("C4+C5"); // 37.51 + s.Row().WriteCell("Grand").Skip().WriteFormula("SUM(C2:C6)"); // Includes the subtotals: 106.02 + s.Row().WriteCell(new DateTime(2026, 1, 31)).WriteCell(new DateTime(2026, 1, 1)).WriteFormula("A8-B8"); // 30 days + s.Row().WriteCell(TimeSpan.FromHours(36)).Skip().WriteFormula("A9*24"); // 36 hours + s.Row().WriteCell(true).WriteCell("text").WriteFormula("COUNT(A10:B10)").WriteFormula("COUNTA(A10:B10)"); // 0, 2 + }); + + using (var doc = OpenValidated(bytes)) { } + var rows = ReadRows(bytes, "Lines"); + Assert.Equal(19.0, rows[2][2]); + Assert.Equal(9.5, rows[2][3]); + Assert.Equal(12.0, rows[3][2]); + Assert.Null(rows[3][3]); + Assert.Equal(31.0, rows[4][2]); + Assert.Equal(6.51, (double)rows[5][2]!, 10); + Assert.Equal(37.51, (double)rows[6][2]!, 10); + Assert.Equal(106.02, (double)rows[7][2]!, 10); + Assert.Equal(30.0, rows[8][2]); + Assert.Equal(36.0, rows[9][2]); + Assert.Equal(0.0, rows[10][2]); + Assert.Equal(2.0, rows[10][3]); + } + + [Fact] + public void ExplicitCachedValues_AreKeptAsGiven_AndVisibleToLaterFormulas() + { + var bytes = ExcelPackage.CreateBytes(wb => + { + var s = wb.Worksheets.Add("S"); + s.CacheFormulaResults(); + s.Row().WriteFormula("SOMETHING()", 5.0); // The caller knows better; the evaluator is not consulted. + s.Row().WriteFormula("\"a\"&\"b\"", "ab"); + s.Row().WriteFormula("A1*2"); // Sees the explicit 5. + s.Row().WriteFormula("COUNTA(A1:A2)"); // The text result counts as content. + }); + var rows = ReadRows(bytes, "S"); + Assert.Equal(5.0, rows[1][0]); + Assert.Equal("ab", rows[2][0]); + Assert.Equal(10.0, rows[3][0]); + Assert.Equal(2.0, rows[4][0]); + } + + [Fact] + public void CellsMode_EvaluatesAtFlush_InAnyWriteOrder() + { + var bytes = ExcelPackage.CreateBytes(wb => + { + var s = wb.Worksheets.Add("Grid"); + s.Cells["A4"].Formula = "SUM(A1:A3)"; // Formulas may be set before their inputs exist. + s.Cells["B4"].Formula = "A4*2"; + s.Cells["A5"].Formula = "SUM(A1:A4)"; // Includes the subtotal: 24. + s.Cells["A6"].Formula = "SUM(A1:A7)"; // Forward reference: no value. + s.Cells["A2"].Value = 7; + s.Cells["A1"].Value = 5; + s.Cells["A3"].Value = "n/a"; + s.CacheFormulaResults(); // Nothing is written until save, so any time before it is fine. + }); + + using (var doc = OpenValidated(bytes)) { } + var rows = ReadRows(bytes, "Grid"); + Assert.Equal(12.0, rows[4][0]); + Assert.Equal(24.0, rows[4][1]); + Assert.Equal(24.0, rows[5][0]); + Assert.Null(rows[6][0]); + } + + [Fact] + public void EditMode_SeesLoadedValues_IncludingCachedFormulaResults() + { + var original = ExcelPackage.CreateBytes(wb => + { + var s = wb.Worksheets.Add("Data"); + s.Row().WriteCell("Sku").WriteCell("Amount").WriteCell("Flag"); + s.Row().WriteCell("A").WriteCell(10.5).WriteCell(true); + s.Row().WriteCell("B").WriteCell(20.25).WriteCell("x"); + s.Row().WriteCell("Total").WriteFormula("SUM(B2:B3)", 30.75); + s.Row().WriteCell("Open").WriteFormula("SUM(B2:B3)"); // No cached value in the original. + }); + + using var output = new MemoryStream(); + using (var package = ExcelPackage.Edit(new MemoryStream(original), output, leaveOpen: true)) + { + var s = package.Workbook.Worksheets.Edit("Data"); + s.CacheFormulaResults(); + s.Cells["B6"].Formula = "SUM(B2:B4)"; // Loaded numbers plus the loaded cached result: 61.5. + s.Cells["C6"].Formula = "COUNTA(C2:C3)"; // Loaded boolean and text: 2. + s.Cells["B7"].Formula = "SUM(B2:B5)"; // B5 is a formula without a known result: no value. + package.Save(); + } + var bytes = output.ToArray(); + using (var doc = OpenValidated(bytes)) { } + var rows = ReadRows(bytes, "Data"); + Assert.Equal(61.5, rows[6][1]); + Assert.Equal(2.0, rows[6][2]); + Assert.Null(rows[7][1]); + Assert.Equal(30.75, rows[4][1]); // The original cached value is untouched. + } + + [Fact] + public void TemplateMode_TreatsTemplateRowsAsOpaque() + { + var template = ExcelPackage.CreateBytes(wb => + { + var s = wb.Worksheets.Add("Lines"); + s.Row(1).WriteCell("Qty"); + s.Row(2).WriteCell(5); // A value the filler never sees. + }); + + using var output = new MemoryStream(); + using (var package = ExcelPackage.OpenTemplate(new MemoryStream(template), output, leaveOpen: true)) + { + var s = package.Workbook.Worksheets.Open("Lines"); + s.CacheFormulaResults(); + s.Row(3).WriteCell(7); + s.Row(4).WriteCell(8); + s.Row(5).WriteFormula("SUM(A3:A4)"); // Only written rows: 15. + s.Row(6).WriteFormula("SUM(A2:A4)"); // Touches template row 2: no value rather than a wrong 15. + package.Save(); + } + var bytes = output.ToArray(); + using (var doc = OpenValidated(bytes)) { } + var rows = ReadRows(bytes, "Lines"); + Assert.Equal(5.0, rows[2][0]); + Assert.Equal(15.0, rows[5][0]); + Assert.Null(rows[6][0]); + } + + [Fact] + public void CacheFormulaResults_MustBeEnabledBeforeTheFirstRow_AndCanBeTurnedOff() + { + using var package = new ExcelPackage(new MemoryStream()); + var late = package.Workbook.Worksheets.Add("Late"); + late.Row().WriteCell(1); + var ex = Assert.Throws(() => late.CacheFormulaResults()); + Assert.Contains("before its first row", ex.Message); + late.CacheFormulaResults(false); // Turning off is always allowed. + + var early = package.Workbook.Worksheets.Add("Early"); + Assert.Same(early, early.CacheFormulaResults()); + Assert.Same(early, early.CacheFormulaResults()); // Idempotent. + Assert.True(early.IsCachingFormulaResults); + early.CacheFormulaResults(false); + Assert.False(early.IsCachingFormulaResults); + } + + [Fact] + public void LoadFromCollection_ValuesAreSeen() + { + var items = Enumerable.Range(1, 10).Select(i => new Line { Name = "L" + i, Amount = i * 2.5 }).ToList(); + var bytes = ExcelPackage.CreateBytes(wb => + { + var s = wb.Worksheets.Add("Loaded"); + s.CacheFormulaResults(); + s.LoadFromCollection(items); + s.Row().WriteCell("Total").WriteFormula("SUM(B2:B11)"); + }); + var rows = ReadRows(bytes, "Loaded"); + Assert.Equal(137.5, rows[12][1]); + } + + private sealed class Line + { + public string Name { get; set; } = ""; + public double Amount { get; set; } + } +} diff --git a/tests/SheetStream.Tests/PivotTableTests.cs b/tests/SheetStream.Tests/PivotTableTests.cs new file mode 100644 index 0000000..9942d12 --- /dev/null +++ b/tests/SheetStream.Tests/PivotTableTests.cs @@ -0,0 +1,373 @@ +// The Open XML SDK exposes nearly every member as nullable; in test assertions a null is a failure anyway. +#pragma warning disable CS8602, CS8603, CS8604 +using System.IO.Compression; +using System.Linq; +using DocumentFormat.OpenXml; +using DocumentFormat.OpenXml.Packaging; +using DocumentFormat.OpenXml.Validation; +using X = DocumentFormat.OpenXml.Spreadsheet; + +namespace SheetStream.Tests; + +/// +/// Pivot tables: package structure (parts, relationships, content types, workbook cache list), schema validity, +/// field mapping errors, the computed location for each field combination, and a round trip of the cache +/// definition through the Open XML SDK. Whether Excel really refreshes the result is covered by +/// tests/excel-check-pivot.ps1, which needs an Excel installation. +/// +public class PivotTableTests +{ + private static readonly string[] Headers = { "Region", "Category", "Product", "Amount", "Qty" }; + + /// Writes a small sales sheet and returns its range so the pivot can point at it. + private static string WriteSales(Workbook wb, string sheetName = "Sales") + { + WriteSalesCore(wb, sheetName, null); + return "A1:E13"; + } + + /// Writes the same sales sheet as an Excel table and returns the table. + private static ExcelTable WriteSalesTable(Workbook wb, string tableName) => WriteSalesCore(wb, "Sales", tableName)!; + + private static ExcelTable? WriteSalesCore(Workbook wb, string sheetName, string? tableName) + { + var s = wb.Worksheets.Add(sheetName); + var table = tableName == null ? null : s.AddTable(tableName, "A1:E13"); // Before the header row, so the names are captured from it. + var header = s.Row(); + foreach (var h in Headers) header.WriteCell(h); + var regions = new[] { "North", "South" }; + var categories = new[] { "Hardware", "Software" }; + for (var i = 0; i < 12; i++) + s.Row().WriteCell(regions[i % 2]).WriteCell(categories[i / 6]).WriteCell("P" + (i % 4)).WriteCell(100 + i * 10.5).WriteCell(i + 1); + return table; + } + + private static SpreadsheetDocument OpenValidated(byte[] bytes) + { + var doc = SpreadsheetDocument.Open(new MemoryStream(bytes), false); + var errors = new OpenXmlValidator(FileFormatVersions.Office2016).Validate(doc).ToList(); + Assert.True(errors.Count == 0, string.Join(Environment.NewLine, errors.Select(e => $"{e.Path?.PartUri} {e.Path?.XPath}: {e.Description}"))); + return doc; + } + + private static string ReadEntry(byte[] bytes, string name) + { + using var zip = new ZipArchive(new MemoryStream(bytes), ZipArchiveMode.Read); + var entry = zip.GetEntry(name); + Assert.True(entry != null, $"Part {name} is missing. Parts: {string.Join(", ", zip.Entries.Select(e => e.FullName))}"); + using var reader = new StreamReader(entry.Open()); + return reader.ReadToEnd(); + } + + [Fact] + public void RangeSource_WritesAllPartsRelationshipsAndContentTypes() + { + var bytes = ExcelPackage.CreateBytes(wb => + { + var range = WriteSales(wb); + var p = wb.Worksheets.Add("Pivot").AddPivotTable("SalesByRegion", "Sales", range, 1, 1, Headers); + p.AddRowField("Region").AddColumnField("Category").AddDataField("Amount", numberFormat: "#,##0.0 \"units\""); + }); + + using var doc = OpenValidated(bytes); + var wbPart = doc.WorkbookPart; + + // Workbook: one cache listed, related to the definition part. + var caches = wbPart.Workbook.GetFirstChild().Elements().ToList(); + Assert.Single(caches); + var cacheDef = (PivotTableCacheDefinitionPart)wbPart.GetPartById(caches[0].Id); + Assert.Equal("/xl/pivotCache/pivotCacheDefinition1.xml", cacheDef.Uri.OriginalString); + Assert.NotNull(cacheDef.PivotTableCacheRecordsPart); + Assert.Equal("/xl/pivotCache/pivotCacheRecords1.xml", cacheDef.PivotTableCacheRecordsPart.Uri.OriginalString); + + // Sheet: the pivot part hangs off the pivot sheet only, and points back to the same cache. + var pivotSheet = wbPart.Workbook.Sheets.Elements().Single(s => s.Name == "Pivot"); + var pivotSheetPart = (WorksheetPart)wbPart.GetPartById(pivotSheet.Id); + var pivotPart = pivotSheetPart.PivotTableParts.Single(); + Assert.Equal("/xl/pivotTables/pivotTable1.xml", pivotPart.Uri.OriginalString); + Assert.Same(cacheDef, pivotPart.PivotTableCacheDefinitionPart); + var salesSheet = wbPart.Workbook.Sheets.Elements().Single(s => s.Name == "Sales"); + Assert.Empty(((WorksheetPart)wbPart.GetPartById(salesSheet.Id)).PivotTableParts); + + // Content types for the three part kinds. + var contentTypes = ReadEntry(bytes, "[Content_Types].xml"); + Assert.Contains("PartName=\"/xl/pivotTables/pivotTable1.xml\" ContentType=\"application/vnd.openxmlformats-officedocument.spreadsheetml.pivotTable+xml\"", contentTypes); + Assert.Contains("PartName=\"/xl/pivotCache/pivotCacheDefinition1.xml\" ContentType=\"application/vnd.openxmlformats-officedocument.spreadsheetml.pivotCacheDefinition+xml\"", contentTypes); + Assert.Contains("PartName=\"/xl/pivotCache/pivotCacheRecords1.xml\" ContentType=\"application/vnd.openxmlformats-officedocument.spreadsheetml.pivotCacheRecords+xml\"", contentTypes); + + // Definition content. + var def = pivotPart.PivotTableDefinition; + Assert.Equal("SalesByRegion", def.Name.Value); + Assert.Equal(caches[0].CacheId.Value, def.CacheId.Value); + Assert.Equal("Values", def.DataCaption.Value); + var fields = def.PivotFields.Elements().ToList(); + Assert.Equal(5, fields.Count); + Assert.Equal(X.PivotTableAxisValues.AxisRow, fields[0].Axis.Value); + Assert.Equal(X.PivotTableAxisValues.AxisColumn, fields[1].Axis.Value); + Assert.True(fields[3].DataField.Value); + Assert.Equal(0, def.RowFields.Elements().Single().Index.Value); + Assert.Equal(1, def.ColumnFields.Elements().Single().Index.Value); + var data = def.DataFields.Elements().Single(); + Assert.Equal("Sum of Amount", data.Name.Value); + Assert.Equal(3u, data.Field.Value); + Assert.Equal(X.DataConsolidateFunctionValues.Sum, data.Subtotal.Value); + // The custom number format landed in styles.xml under the id the data field references. + var numFmt = wbPart.WorkbookStylesPart.Stylesheet.NumberingFormats.Elements().Single(n => n.NumberFormatId.Value == data.NumberFormatId.Value); + Assert.Equal("#,##0.0 \"units\"", numFmt.FormatCode.Value); + Assert.Equal("PivotStyleMedium9", def.PivotTableStyle.Name.Value); + } + + [Fact] + public void CacheDefinition_RoundTrips_RefreshOnLoadAndEmptyRecords() + { + var bytes = ExcelPackage.CreateBytes(wb => + { + var range = WriteSales(wb); + wb.Worksheets.Add("Pivot").AddPivotTable("P", "Sales", range, 1, 1, Headers).AddRowField("Region").AddDataField("Qty", PivotFunction.Count); + }); + + using var doc = OpenValidated(bytes); + var cacheDef = doc.WorkbookPart.PivotTableCacheDefinitionParts.Single().PivotCacheDefinition; + Assert.True(cacheDef.RefreshOnLoad.Value); + Assert.False(cacheDef.SaveData.Value); + Assert.Equal(0u, cacheDef.RecordCount.Value); + Assert.Equal("SheetStream", cacheDef.RefreshedBy.Value); + Assert.Equal(6, (int)cacheDef.CreatedVersion.Value); + Assert.Equal(6, (int)cacheDef.RefreshedVersion.Value); + Assert.Equal(3, (int)cacheDef.MinRefreshableVersion.Value); + var source = cacheDef.CacheSource.WorksheetSource; + Assert.Equal("A1:E13", source.Reference.Value); + Assert.Equal("Sales", source.Sheet.Value); + Assert.Null(source.Name); + Assert.Equal(Headers, cacheDef.CacheFields.Elements().Select(f => f.Name.Value)); + Assert.All(cacheDef.CacheFields.Elements(), f => Assert.Empty(f.SharedItems.ChildElements)); + var records = doc.WorkbookPart.PivotTableCacheDefinitionParts.Single().PivotTableCacheRecordsPart.PivotCacheRecords; + Assert.Equal(0u, records.Count.Value); + Assert.Empty(records.ChildElements); + } + + [Fact] + public void TableSource_UsesTableNameAndItsColumns() + { + var bytes = ExcelPackage.CreateBytes(wb => + { + var table = WriteSalesTable(wb, "SalesTbl"); + var pivot = wb.Worksheets.Add("Pivot").AddPivotTable("FromTable", table, 2, 2); + Assert.Equal(Headers, pivot.Headers); // Captured from the header row that was written after AddTable. + pivot.AddRowField("product").AddDataField("Amount", PivotFunction.Average, "Avg amount"); + }); + + using var doc = OpenValidated(bytes); + var source = doc.WorkbookPart.PivotTableCacheDefinitionParts.Single().PivotCacheDefinition.CacheSource.WorksheetSource; + Assert.Equal("SalesTbl", source.Name.Value); + Assert.Null(source.Reference); + var def = doc.WorkbookPart.WorksheetParts.SelectMany(p => p.PivotTableParts).Single().PivotTableDefinition; + Assert.Equal(2, def.RowFields.Elements().Single().Index.Value); // "product" matched case-insensitively. + Assert.Equal("Avg amount", def.DataFields.Elements().Single().Name.Value); + Assert.Equal(X.DataConsolidateFunctionValues.Average, def.DataFields.Elements().Single().Subtotal.Value); + } + + [Fact] + public void TableSource_OnSameSheet_NeedsHeadersFirst() + { + using var package = new ExcelPackage(new MemoryStream()); + var s = package.Workbook.Worksheets.Add("Sales"); + var table = s.AddTable("Tbl", "A1:E13"); + // The header row is not written yet, so the field names are unknown. + var ex = Assert.Throws(() => s.AddPivotTable("P", table, 1, 8)); + Assert.Contains("header", ex.Message); + var header = s.Row(); + foreach (var h in Headers) header.WriteCell(h); + var pivot = s.AddPivotTable("P", table, 1, 8); + Assert.Equal(Headers, pivot.Headers); + pivot.AddRowField("Region").AddDataField("Amount"); // So the package can save when disposed. + for (var i = 0; i < 12; i++) s.Row().WriteCell("N").WriteCell("C").WriteCell("P").WriteCell(1.0).WriteCell(1); + } + + [Fact] + public void TwoPivots_GetUniqueCacheIdsAndPartNumbers() + { + var bytes = ExcelPackage.CreateBytes(wb => + { + var range = WriteSales(wb); + var sheet = wb.Worksheets.Add("Pivots"); + sheet.AddPivotTable("A", "Sales", range, 1, 1, Headers).AddRowField("Region").AddDataField("Amount"); + sheet.AddPivotTable("B", "Sales", range, 1, 8, Headers).AddRowField("Category").AddDataField("Qty"); + // A third one on another sheet, sourcing the same range. + wb.Worksheets.Add("More").AddPivotTable("C", "Sales", range, 1, 1, Headers).AddFilterField("Region").AddRowField("Product").AddDataField("Amount"); + }); + + using var doc = OpenValidated(bytes); + var caches = doc.WorkbookPart.Workbook.GetFirstChild().Elements().Select(c => c.CacheId.Value).ToList(); + Assert.Equal(3, caches.Count); + Assert.Equal(3, caches.Distinct().Count()); + var defs = doc.WorkbookPart.WorksheetParts.SelectMany(p => p.PivotTableParts).ToList(); + Assert.Equal(3, defs.Count); + Assert.Equal(caches.OrderBy(c => c), defs.Select(d => d.PivotTableDefinition.CacheId.Value).OrderBy(c => c)); + Assert.Equal(3, doc.WorkbookPart.PivotTableCacheDefinitionParts.Count()); + Assert.Equal(new[] { "/xl/pivotTables/pivotTable1.xml", "/xl/pivotTables/pivotTable2.xml", "/xl/pivotTables/pivotTable3.xml" }, defs.Select(d => d.Uri.OriginalString).OrderBy(u => u)); + // The definition is the right one for each sheet: pivot C carries the page field. + var c = defs.Single(d => d.PivotTableDefinition.Name.Value == "C").PivotTableDefinition; + Assert.Equal(0, c.PageFields.Elements().Single().Field.Value); + Assert.Equal(1u, c.Location.RowPageCount.Value); + } + + [Fact] + public void SameSheetSource_IsAllowed() + { + var bytes = ExcelPackage.CreateBytes(wb => + { + var range = WriteSales(wb); + // The pivot lives next to its own data. + wb.Worksheets.Current.AddPivotTable("Inline", "sales", range, 1, 8, Headers).AddRowField("Region").AddDataField("Amount"); + }); + using var doc = OpenValidated(bytes); + Assert.Equal("Sales", doc.WorkbookPart.PivotTableCacheDefinitionParts.Single().PivotCacheDefinition.CacheSource.WorksheetSource.Sheet.Value); + } + + [Fact] + public void UnknownHeader_ThrowsNamingAvailableHeaders() + { + using var package = new ExcelPackage(new MemoryStream()); + var range = WriteSales(package.Workbook); + var pivot = package.Workbook.Worksheets.Add("Pivot").AddPivotTable("P", "Sales", range, 1, 1, Headers); + var ex = Assert.Throws(() => pivot.AddRowField("Reggion")); + Assert.Contains("Reggion", ex.Message); + Assert.Contains("Region, Category, Product, Amount, Qty", ex.Message); + Assert.Throws(() => pivot.AddDataField("Total")); + pivot.AddDataField("Amount"); // So the package can save when disposed. + } + + [Fact] + public void ArgumentErrors_AreReported() + { + using var package = new ExcelPackage(new MemoryStream()); + var wb = package.Workbook; + var range = WriteSales(wb); + var sheet = wb.Worksheets.Add("Pivot"); + Assert.Throws(() => sheet.AddPivotTable("P", "Missing", range, 1, 1, Headers)); // Unknown source sheet. + Assert.Throws(() => sheet.AddPivotTable("P", "Sales", range, 1, 1, new[] { "A", "B" })); // Header count mismatch. + Assert.Throws(() => sheet.AddPivotTable("P", "Sales", "A1:E1", 1, 1, Headers)); // No data rows. + Assert.Throws(() => sheet.AddPivotTable("P", "Sales", range, 1, 1, new[] { "A", "A", "C", "D", "E" })); // Duplicate headers. + var pivot = sheet.AddPivotTable("P", "Sales", range, 1, 1, Headers); + Assert.Throws(() => sheet.AddPivotTable("p", "Sales", range, 20, 1, Headers)); // Name taken (case-insensitive). + Assert.Throws(() => sheet.AddPivotTable("Q", "Sales", range, 0, 1, Headers)); // Row out of range. + // A data field named like a source column makes Excel fail with "field name already exists". + var ex = Assert.Throws(() => pivot.AddDataField("Amount", displayName: "Qty")); + Assert.Contains("Qty", ex.Message); + pivot.AddDataField("Amount"); + Assert.Throws(() => pivot.AddDataField("Amount")); // Same caption twice. + pivot.AddDataField("Amount", PivotFunction.Count); // Different caption: fine. + } + + [Fact] + public void PivotWithoutFields_FailsAtSave() + { + using var package = new ExcelPackage(new MemoryStream()); + var range = WriteSales(package.Workbook); + package.Workbook.Worksheets.Add("Pivot").AddPivotTable("Empty", "Sales", range, 1, 1, Headers); + var ex = Assert.Throws(() => package.Save()); + Assert.Contains("no fields", ex.Message); + } + + [Fact] + public void TemplateSheet_RefusesPivots() + { + var template = ExcelPackage.CreateBytes(wb => wb.Worksheets.Add("Data").Row().WriteCell("x")); + using var package = ExcelPackage.OpenTemplate(new MemoryStream(template), new MemoryStream()); + var sheet = package.Workbook.Worksheets.Open("Data"); + Assert.Throws(() => sheet.AddPivotTable("P", "Data", "A1:A2", 5, 1, new[] { "x" })); + } + + /// + /// The location element as Excel writes it for a fresh pivot: body ref sized for one item per axis plus + /// headers and grand totals, header/data offsets per field combination. Excel resizes it on refresh. + /// + [Theory] + [InlineData("row", 3, 1, "A3:B5", 1, 1, 1)] // "Row Labels | Sum" header, one item, grand total. + [InlineData("row,col", 3, 1, "A3:C6", 1, 2, 1)] // Caption row + header row, one item, grand total; item column + total column. + [InlineData("row,col,data2", 3, 1, "A3:E7", 1, 3, 1)] // Values row added; two data columns per item and per total. + [InlineData("row,data2", 3, 1, "A3:C5", 1, 1, 1)] // Values side by side without column fields: still one header row. + [InlineData("page,row", 3, 1, "A5:B7", 1, 1, 1)] // One filter row + blank row push the body down two rows. + [InlineData("page,page,row,col", 1, 2, "B4:D7", 1, 2, 1)] // Two filters: body starts at row 4, column B. + [InlineData("data", 3, 1, "A3:A4", 1, 1, 0)] // Caption over a single value, no row axis. + [InlineData("row,nogrand", 3, 1, "A3:B4", 1, 1, 1)] // No grand total row. + [InlineData("row,col,nogrand", 3, 1, "A3:B5", 1, 2, 1)] // No grand total row or column. + [InlineData("row,row,outline", 3, 1, "A3:C5", 1, 1, 2)] // Outline form: one column per row field. + public void Location_IsComputedForEachFieldCombination(string spec, int row, int column, string expectedRef, int headerRow, int dataRow, int dataCol) + { + var bytes = ExcelPackage.CreateBytes(wb => + { + var range = WriteSales(wb); + var p = wb.Worksheets.Add("Pivot").AddPivotTable("P", "Sales", range, row, column, Headers); + var rowFields = new Queue(new[] { "Region", "Category", "Product" }); + var dataAdded = false; + foreach (var part in spec.Split(',')) + { + switch (part) + { + case "row": p.AddRowField(rowFields.Dequeue()); break; + case "col": p.AddColumnField("Qty"); break; + case "page": p.AddFilterField(rowFields.Dequeue()); break; + case "data2": p.AddDataField("Amount"); p.AddDataField("Amount", PivotFunction.Count); dataAdded = true; break; + case "data": p.AddDataField("Amount"); dataAdded = true; break; + case "nogrand": p.ShowRowGrandTotals = false; p.ShowColumnGrandTotals = false; break; + case "outline": p.Compact = false; break; + } + } + if (!dataAdded && !spec.Contains("data")) p.AddDataField("Amount"); + var loc = p.ComputeLocation(); + Assert.Equal(expectedRef, loc.Ref.ToString()); + }); + + using var doc = OpenValidated(bytes); + var location = doc.WorkbookPart.WorksheetParts.SelectMany(p => p.PivotTableParts).Single().PivotTableDefinition.Location; + Assert.Equal(expectedRef, location.Reference.Value); + Assert.Equal((uint)headerRow, location.FirstHeaderRow.Value); + Assert.Equal((uint)dataRow, location.FirstDataRow.Value); + Assert.Equal((uint)dataCol, location.FirstDataColumn.Value); + } + + [Fact] + public void TwoDataFields_PutValuesAxisOnColumns() + { + var bytes = ExcelPackage.CreateBytes(wb => + { + var range = WriteSales(wb); + var p = wb.Worksheets.Add("Pivot").AddPivotTable("P", "Sales", range, 1, 1, Headers); + p.ShowRowGrandTotals = false; + p.ShowRowStripes = true; + p.AddRowField("Region").AddDataField("Amount").AddDataField("Qty", PivotFunction.Max, numberFormat: "0"); + }); + using var doc = OpenValidated(bytes); + var def = doc.WorkbookPart.WorksheetParts.SelectMany(p => p.PivotTableParts).Single().PivotTableDefinition; + Assert.Equal(-2, def.ColumnFields.Elements().Single().Index.Value); // The "Values" pseudo-field. + Assert.Equal(2, def.DataFields.Count()); + Assert.False(def.RowGrandTotals.Value); + Assert.Null(def.ColumnGrandTotals); // Default (true) is not written. + Assert.True(def.PivotTableStyle.ShowRowStripes.Value); + // Built-in format "0" maps to numFmtId 1 without a custom numFmt. + Assert.Equal(1u, def.DataFields.Elements().Last().NumberFormatId.Value); + } + + [Fact] + public void LoadResult_ExposesHeadersRangeAndTable() + { + ExcelPackage.CreateBytes(wb => + { + var s = wb.Worksheets.Add("Data"); + s.Row().WriteCell("Title"); + var plain = s.LoadFromCollection(new[] { new { Name = "a", Value = 1 }, new { Name = "b", Value = 2 } }, new LoadOptions { StartColumn = 2 }); + Assert.Equal(new[] { "Name", "Value" }, plain.Headers); + Assert.Equal("B2:C4", plain.Range); + Assert.Null(plain.Table); + var withTable = s.LoadFromCollection(new[] { new { Name = "c", Value = 3 } }, new LoadOptions { TableName = "T" }); + Assert.NotNull(withTable.Table); + Assert.Equal("T", withTable.Table.Name); + Assert.Equal("A5:B6", withTable.Range); + var empty = s.LoadFromCollection(Array.Empty>()); + Assert.Empty(empty.Headers); + Assert.Equal("A7", empty.Range); + }); + } +} diff --git a/tests/SheetStream.Tests/ReaderFuzzTests.cs b/tests/SheetStream.Tests/ReaderFuzzTests.cs index 2d7ca90..653ddad 100644 --- a/tests/SheetStream.Tests/ReaderFuzzTests.cs +++ b/tests/SheetStream.Tests/ReaderFuzzTests.cs @@ -53,6 +53,21 @@ private static void ReadEverything(byte[] bytes) _ = row.GetString(c); } + [Fact] + public void NegativeStyleIndex_ReadsAsPlainValue() + { + // Regression: s="-1" passed the "< Count" guard and indexed the style table at -1. Found by the fuzzer in CI. + // Cells that already carry a style keep a single (now negative) s attribute; the rest get one added. + var mutated = MutatePart(Seed(), "xl/worksheets/sheet1.xml", text => + System.Text.RegularExpressions.Regex.Replace( + System.Text.RegularExpressions.Regex.Replace(text, "(]*?\\bs=\")\\d+(\")", "$1-1$2"), + "]*\\bs=)", " r.Values.Any(v => v != null)); + } + [Theory] [InlineData("xl/worksheets/sheet1.xml")] [InlineData("xl/sharedStrings.xml")] @@ -62,7 +77,11 @@ private static void ReadEverything(byte[] bytes) public async Task MutatedXml_FailsCleanlyOrReads(string part) { var seed = Seed(); - var rng = new Random(part.GetHashCode() ^ 12345); // Deterministic per part, so failures reproduce. + // Deterministic per part, so failures reproduce. string.GetHashCode() is randomised per process on + // .NET Core, so a stable FNV-1a hash is used instead; the first run with a random seed found the s="-1" bug. + var seedHash = 2166136261u; + foreach (var ch in part) seedHash = (seedHash ^ ch) * 16777619u; + var rng = new Random(unchecked((int)seedHash) ^ 12345); // Fragments that stress the parser: huge or negative numbers, bad references, broken structure. string[] junk = { "<", ">", "\"", "999999999999", "-1", "ZZZZZ1", "XFE1", "&", "", "", "t=\"s\"", "s=\"99999\"", "r=\"A0\"", "\u0000", "1E400" }; diff --git a/tests/excel-check-pivot.ps1 b/tests/excel-check-pivot.ps1 new file mode 100644 index 0000000..4fb0f36 --- /dev/null +++ b/tests/excel-check-pivot.ps1 @@ -0,0 +1,110 @@ +<# +.SYNOPSIS + Opens an .xlsx in the installed Excel (COM automation), refreshes every pivot table and prints its cells. + +.DESCRIPTION + The Open XML validator proves a file is schema-correct, not that Excel accepts it. This script is the + second gate for pivot tables: Excel must open the workbook without repairing it (a repaired workbook + silently drops the pivot parts, so the pivot count is compared with -ExpectedPivots), every pivot must + refresh from its source (RefreshTable throws when Excel rejects the cache), and the refreshed values are + printed as a table so the sums can be checked by eye or by a caller. + + Exit code 0 = everything refreshed, 1 = a check failed. Excel is always closed, even on failure. + +.PARAMETER Path + The workbook to check. + +.PARAMETER ExpectedPivots + Total number of pivot tables the workbook should contain across all sheets. -1 skips the count check. + +.EXAMPLE + .\excel-check-pivot.ps1 -Path .\Showcase.xlsx -ExpectedPivots 1 +#> +param( + [Parameter(Mandatory = $true)][string]$Path, + [int]$ExpectedPivots = -1 +) + +$ErrorActionPreference = 'Stop' +$fullPath = (Resolve-Path -LiteralPath $Path).Path +$excel = $null +$workbook = $null +$failed = $false + +function Write-RangeTable { + param($Range) + # Value2 returns a 2-D array for several cells, a scalar for one cell. + $values = $Range.Value2 + $rows = $Range.Rows.Count + $cols = $Range.Columns.Count + if ($rows -eq 1 -and $cols -eq 1) { Write-Output (" " + $values); return } + for ($r = 1; $r -le $rows; $r++) { + $cells = @() + for ($c = 1; $c -le $cols; $c++) { + $v = $values[$r, $c] + if ($null -eq $v) { $v = '' } + elseif ($v -is [double]) { $v = $v.ToString('0.##', [System.Globalization.CultureInfo]::InvariantCulture) } + $cells += ('{0,-18}' -f $v) + } + Write-Output (" " + ($cells -join ' | ')) + } +} + +try { + $excel = New-Object -ComObject Excel.Application + $excel.Visible = $false + $excel.DisplayAlerts = $false # No repair / compatibility dialogs; a damaged file then fails to open or loses parts. + $excel.AskToUpdateLinks = $false + + Write-Output "Opening $fullPath in Excel $($excel.Version)" + $workbook = $excel.Workbooks.Open($fullPath, 0, $true) # UpdateLinks=0, ReadOnly=true + $readOnly = $workbook.ReadOnly + $sheetCount = $workbook.Worksheets.Count + Write-Output "Opened. ReadOnly=$readOnly Sheets=$sheetCount" + + $total = 0 + foreach ($ws in $workbook.Worksheets) { + $pivots = $ws.PivotTables() + if ($pivots.Count -eq 0) { continue } + Write-Output ("Sheet '{0}': {1} pivot table(s)" -f $ws.Name, $pivots.Count) + for ($i = 1; $i -le $pivots.Count; $i++) { + $pt = $pivots.Item($i) + $total++ + Write-Output (" Pivot '{0}' source={1} cache records before refresh={2}" -f $pt.Name, $pt.SourceData, $pt.PivotCache().RecordCount) + try { + $ok = $pt.RefreshTable() + Write-Output (" RefreshTable returned {0}; records after refresh={1}; range={2}" -f $ok, $pt.PivotCache().RecordCount, $pt.TableRange2.Address(0, 0)) + Write-RangeTable -Range $pt.TableRange2 + } + catch { + $failed = $true + Write-Output (" FAILED: Excel refused to refresh pivot '{0}': {1}" -f $pt.Name, $_.Exception.Message) + } + } + } + + if ($ExpectedPivots -ge 0 -and $total -ne $ExpectedPivots) { + $failed = $true + Write-Output ("FAILED: expected {0} pivot table(s) but Excel sees {1}. Excel probably repaired the file and dropped the pivot parts." -f $ExpectedPivots, $total) + } + elseif ($total -eq 0) { + $failed = $true + Write-Output "FAILED: the workbook contains no pivot tables." + } +} +catch { + $failed = $true + Write-Output ("FAILED: {0}" -f $_.Exception.Message) +} +finally { + if ($null -ne $workbook) { try { $workbook.Close($false) } catch { } } + if ($null -ne $excel) { try { $excel.Quit() } catch { } } + if ($null -ne $workbook) { [void][System.Runtime.InteropServices.Marshal]::ReleaseComObject($workbook) } + if ($null -ne $excel) { [void][System.Runtime.InteropServices.Marshal]::ReleaseComObject($excel) } + [GC]::Collect() + [GC]::WaitForPendingFinalizers() +} + +if ($failed) { Write-Output "RESULT: FAILED"; exit 1 } +Write-Output "RESULT: OK - every pivot table refreshed in Excel" +exit 0