diff --git a/.github/workflows/benchmark-pr.yml b/.github/workflows/benchmark-pr.yml index 059d97da..ecbbcad9 100644 --- a/.github/workflows/benchmark-pr.yml +++ b/.github/workflows/benchmark-pr.yml @@ -22,7 +22,8 @@ jobs: runs-on: ubuntu-latest outputs: benchmarks: ${{ steps.filter.outputs.benchmarks }} - run: ${{ steps.filter.outputs.benchmarks == 'true' && (github.base_ref == 'master' || contains(github.event.pull_request.labels.*.name, 'run-benchmarks')) }} + # Opt-in on every base branch: the suite is ~40 runner-minutes, too much to repeat on each push. + run: ${{ steps.filter.outputs.benchmarks == 'true' && contains(github.event.pull_request.labels.*.name, 'run-benchmarks') }} steps: - name: Checkout uses: actions/checkout@v7 @@ -66,7 +67,7 @@ jobs: cache-dependency-path: '**/*.csproj' - name: Build (Release) - run: dotnet build tests/ExcelReader.Benchmarks/ExcelReader.Benchmarks.csproj --configuration Release + run: dotnet build tests/ExcelReader.Benchmarks/ExcelReader.Benchmarks.csproj --configuration Release -p:RunAnalyzers=false - name: Check group coverage run: bash tests/ExcelReader.Benchmarks/check-benchmark-groups.sh @@ -94,7 +95,7 @@ jobs: run: dotnet restore tests/ExcelReader.Benchmarks/ExcelReader.Benchmarks.csproj - name: Build (Release) - run: dotnet build tests/ExcelReader.Benchmarks/ExcelReader.Benchmarks.csproj --configuration Release --no-restore + run: dotnet build tests/ExcelReader.Benchmarks/ExcelReader.Benchmarks.csproj --configuration Release --no-restore -p:RunAnalyzers=false - name: Run benchmarks run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c994a4e..23f980d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,6 +36,7 @@ jobs: dotnet: - 'src/**' - 'tests/**' + - 'docs/snippets/**' - 'Directory.Build.props' - 'global.json' - 'ExcelReader.slnx' @@ -64,6 +65,12 @@ jobs: os: [ubuntu-latest, windows-latest, macos-latest] env: RUN: ${{ needs.changes.outputs.dotnet }} + # The whole solution compiles the same on every OS, so Linux builds all of it and the others + # build only what the tests need. + BUILD_TARGET: ${{ matrix.os == 'ubuntu-latest' && 'ExcelReader.slnx' || 'tests/ExcelReader.Tests/ExcelReader.Tests.csproj' }} + # Analyzers are about two thirds of the compile and report the same on every OS, so only the + # Linux build runs them; every other build in CI skips them. Source generators still run. + BUILD_ARGS: ${{ matrix.os != 'ubuntu-latest' && '-p:RunAnalyzers=false' || '' }} steps: - name: Nothing to build @@ -84,11 +91,11 @@ jobs: - name: Restore if: env.RUN == 'true' - run: dotnet restore ExcelReader.slnx + run: dotnet restore ${{ env.BUILD_TARGET }} - name: Build (Release) if: env.RUN == 'true' - run: dotnet build ExcelReader.slnx --configuration Release --no-restore -p:DeterministicSourcePaths=false + run: dotnet build ${{ env.BUILD_TARGET }} --configuration Release --no-restore -p:DeterministicSourcePaths=false ${{ env.BUILD_ARGS }} - name: Verify formatting (whitespace) if: env.RUN == 'true' && matrix.os == 'ubuntu-latest' @@ -155,7 +162,7 @@ jobs: cache-dependency-path: '**/*.csproj' - name: Build (Release) - run: dotnet build tests/ExcelReader.Tests/ExcelReader.Tests.csproj --configuration Release + run: dotnet build tests/ExcelReader.Tests/ExcelReader.Tests.csproj --configuration Release -p:RunAnalyzers=false - name: Test (Release) run: dotnet test --project tests/ExcelReader.Tests/ExcelReader.Tests.csproj --configuration Release --no-build @@ -188,7 +195,7 @@ jobs: - name: Pack and verify the generator ships inside the nupkg shell: pwsh run: | - dotnet pack src/ExcelReader.Core/ExcelReader.Core.csproj -c Release -o ./artifacts + dotnet pack src/ExcelReader.Core/ExcelReader.Core.csproj -c Release -o ./artifacts -p:RunAnalyzers=false $pkg = Get-ChildItem ./artifacts/*.nupkg | Select-Object -First 1 Add-Type -AssemblyName System.IO.Compression.FileSystem $zip = [System.IO.Compression.ZipFile]::OpenRead($pkg.FullName) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index b93ef95d..a0b08d88 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -1,8 +1,9 @@ name: CodeQL on: + # develop is covered by its pull request into master; a push trigger there would run everything twice. push: - branches: [ master, develop ] + branches: [ master ] pull_request: branches: [ master, develop ] types: [ opened, synchronize, reopened, ready_for_review ] @@ -89,9 +90,14 @@ jobs: languages: ${{ matrix.language }} queries: security-extended + # Only the shipped projects: tests, benchmarks and the fuzz harness double the build for code + # no one runs against untrusted input. - name: Build (Release) if: matrix.language == 'csharp' - run: dotnet build ExcelReader.slnx --configuration Release + run: | + for project in src/*/*.csproj; do + dotnet build "$project" --configuration Release -p:RunAnalyzers=false + done - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..14b6e992 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,74 @@ +name: Docs -> GitHub Pages + +on: + push: + branches: [master] + paths: &docs-paths + - 'src/ExcelReader.Core/**' + - 'src/ExcelReader.Arrow/**' + - 'src/ExcelReader.Native/include/**' + - 'python/src/**' + - 'python/scripts/api_markdown.py' + - 'cpp/scripts/api_markdown.py' + - 'docs/**' + - '**.md' + - 'docfx.json' + - 'assets/**' + - '.github/workflows/docs.yml' + # Builds without publishing, so a broken page or a Doxygen @param that no longer matches its + # signature fails the pull request instead of the next publish. + pull_request: + paths: *docs-paths + workflow_dispatch: + +concurrency: + group: docs-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: write + +jobs: + docs: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-dotnet@v6 + with: + global-json-file: global.json + + # The snippets themselves are compiled by ci.yml, which builds the whole solution. + - name: Check guide examples match docs/snippets + run: python3 docs/snippets/sync.py --check + + # Importing excelreader loads the native library lazily, so this needs only the sources. + - name: Build Python API reference + run: python3 python/scripts/api_markdown.py + + # Fails on any Doxygen documentation warning, such as an @param naming no parameter. + - name: Build C++ API reference + run: | + sudo apt-get update + sudo apt-get install --yes --no-install-recommends doxygen + python3 cpp/scripts/api_markdown.py + + # Fails on a broken page link or an xref naming a member that no longer exists. Core loads the + # generator as an analyzer from its bin/Release output, so it is built first, or docfx warns + # FailedToResolveAnalyzer. + - name: Build site + run: | + dotnet build src/ExcelReader.Generator/ExcelReader.Generator.csproj --configuration Release -p:RunAnalyzers=false + dotnet tool install --global docfx --version 2.81.0 + docfx docfx.json --warningsAsErrors + + # keep_files leaves dev/bench (benchmark-pages.yml) in place; pages removed from the docs + # stay on the branch until deleted by hand. + - name: Publish to gh-pages + if: github.event_name != 'pull_request' + uses: peaceiris/actions-gh-pages@v4 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: docs/_site + publish_branch: gh-pages + keep_files: true diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index ab0a4f81..d0dd6254 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -53,7 +53,7 @@ jobs: strategy: fail-fast: false matrix: - target: [xlsx, xlsx-memory, xlsx-differential, xlsb, xlsb-memory, xlsb-differential, xls, encrypted, csv, csv-sniff, csv-parallel] + target: [xlsx, xlsx-memory, xlsx-differential, xlsx-pipeline, xlsb, xlsb-memory, xlsb-differential, xlsb-pipeline, xls, encrypted, csv, csv-sniff, csv-parallel, inflate] steps: - name: Checkout uses: actions/checkout@v7 @@ -76,7 +76,7 @@ jobs: - name: Publish harness run: | - dotnet publish tests/ExcelReader.Fuzz/ExcelReader.Fuzz.csproj -c Release -o fuzz-out + dotnet publish tests/ExcelReader.Fuzz/ExcelReader.Fuzz.csproj -c Release -o fuzz-out -p:RunAnalyzers=false chmod +x fuzz-out/ExcelReader.Fuzz - name: Generate seed corpus @@ -110,6 +110,7 @@ jobs: xls) dict=xls ;; encrypted) dict=encrypted ;; csv*) dict=csv ;; + inflate) dict=inflate ;; esac ./libfuzzer-dotnet \ --target_path=fuzz-out/ExcelReader.Fuzz \ @@ -158,7 +159,7 @@ jobs: - name: Build if: env.RUN == 'true' - run: dotnet build tests/ExcelReader.Fuzz/ExcelReader.Fuzz.csproj -c Release + run: dotnet build tests/ExcelReader.Fuzz/ExcelReader.Fuzz.csproj -c Release -p:RunAnalyzers=false - name: Generate corpus if: env.RUN == 'true' diff --git a/.github/workflows/native-bindings.yml b/.github/workflows/native-bindings.yml index 754d53c1..06d6862d 100644 --- a/.github/workflows/native-bindings.yml +++ b/.github/workflows/native-bindings.yml @@ -1,8 +1,9 @@ name: Native bindings on: + # develop is covered by its pull request into master; a push trigger there would run everything twice. push: - branches: [master, develop] + branches: [master] pull_request: branches: [master, develop] types: [opened, synchronize, reopened, ready_for_review] @@ -76,6 +77,8 @@ jobs: - name: Build the native library shell: bash + env: + RunAnalyzers: 'false' run: python python/scripts/build_native.py - name: Upload native library @@ -145,13 +148,17 @@ jobs: EXCELREADER_NATIVE_LIB_DIR: ${{ github.workspace }}/${{ steps.lib.outputs.dir }} run: cargo test --workspace --all-features --verbose + # The next two compile the crate again (default features, then the bench profile); that + # catches nothing platform-specific the test build above missed, so one OS is enough. - name: cargo build (default features) + if: runner.os == 'Linux' working-directory: rust env: EXCELREADER_NATIVE_LIB_DIR: ${{ github.workspace }}/${{ steps.lib.outputs.dir }} run: cargo build --workspace --verbose - name: cargo bench (smoke) + if: runner.os == 'Linux' working-directory: rust env: EXCELREADER_NATIVE_LIB_DIR: ${{ github.workspace }}/${{ steps.lib.outputs.dir }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e7559e70..91fff678 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -312,8 +312,6 @@ jobs: grep -q 'already exists' <<<"$out" || exit 1 fi - # The workspace config points EXCELREADER_NATIVE_LIB_DIR at a local build that this job never makes; - # without it the package verification downloads the native library from the release, as users do. - name: Publish excelreader to crates.io working-directory: rust/excelreader env: diff --git a/.gitignore b/.gitignore index 87734d65..b0eb550c 100644 --- a/.gitignore +++ b/.gitignore @@ -506,3 +506,8 @@ python/src/excelreader/_lib/* build/ # Subagent-driven-development scratch (per-plan ledgers, briefs, review packages) .superpowers/ +# docfx output and the generated Python/C++ API pages (docfx.json, */scripts/api_markdown.py); CI builds and publishes it to gh-pages +docs/_site/ +docs/api/ +docs/python-api/ +docs/cpp-api/ diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 2d7c67de..a627640d 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -6,7 +6,7 @@ directly rather than looking for a commentary alongside it. What the code deliberately does *not* carry is the evidence behind a decision — benchmark numbers, memory-footprint measurements, approaches that were tried and rejected. That lives in -[`docs/`](docs/), and [`STYLEGUIDE.md`](STYLEGUIDE.md#comments) says why. +[`docs/`](docs/README.md), and [`STYLEGUIDE.md`](STYLEGUIDE.md#comments) says why. ## The four format families @@ -47,13 +47,73 @@ source-generated map) or `ExcelParser.Build` (a runtime-configured map). Its mod a `ref struct` model binds directly to `Cell.Value` spans — zero allocation for the container and, for span-typed columns, for the values too. It consumes `Row`/`Cell` from any sheet uniformly. -## Parallel CSV parsing - -`CsvParallel.ParseAsync` (`src/ExcelReader.Core/Parser/ParallelCsv/ParallelCsv*.cs`) is an opt-in -alternate path over the same typed-parsing machinery, for sources large enough that partitioning -pays for itself. `ParallelCsvFactory` decides eligibility — a seekable, UTF-8, large-enough source — -and falls back to the ordinary sequential `ExcelParser` otherwise, so callers get identical output -either way. +## Parallel sheet reading + +`SheetParallel` (`src/ExcelReader.Core/Reader/SheetParallel.cs`) reads one sheet on several threads. It is +opt-in, takes any `IExcelSheet`, validates its arguments and dispatches on the sheet's concrete type: + +| sheet | engine | +|---|---| +| `CsvSheet` | the CSV engine (`ParallelCsvProcessor`, `ParallelCsvEnumerable`), described below. Its source comes from `CsvReader.TryGetChunkSource`, which the native layer also uses. | +| `XlsxSheet`, `XlsbSheet` | the pipeline engine (`src/ExcelReader.Core/Parser/SheetPipeline`) | +| anything else, and every fallback case | one sequential runner over `IExcelRowEnumerator` | + +The fallback cases give the same results with no error: a degree of parallelism of 1, a sheet under 256 KB +uncompressed, XLS, a table view (`ExcelTable.AsSheet`), and a CSV that cannot be partitioned (a stream that is not a file +or a non-UTF-8 encoding). The caller owns the workbook; the engine holds a `ReaderLifetime` lease +until the task completes or the enumeration ends. + +The CSV engine is not folded into a shared partition abstraction. It is tuned, and its model, a static +byte-range plan with speculative starts and re-reads, differs from the pipeline's dynamic, exact chunks. What +the two share is the contract types (`IRowRecord`, `IRowAccumulator`, `RowAggregation`, `RowAction`) and the +binders in `Parser/ParallelCsv` (`MappedProjection`, `MappedAggregation`, `MappedCallback`, `RecordAggregation`, +`RecordCallback`), which pick the parser's CSV type map for a `CsvSheet` and the general one otherwise. + +### The pipeline engine + +``` +zip entry -> producer (1 thread) -> bounded queue -> N workers -> ordered merge + inflate, cut at the row-aligned parse + by chunk index + last row boundary chunks, indexed row action +``` + +Inflate is sequential, so the engine splits the work around it instead of through it. One producer thread +owns the entry stream, opened through `ZipIndex.OpenEntryStream` so the decompressed-byte limits count where +they do on the sequential path. It inflates into pooled buffers and cuts each at the last row boundary, +copying the partial tail into the next buffer; chunks default to 256 KB (`SheetPipelineSource.DefaultChunkBytes`). +In XLSX the boundary is the last ``, +call `Seed()`, then run the row action for each row. Shared strings are loaded before the pipeline starts, and a +prologue reads up to `HeaderRow` with an ordinary enumerator, so header rows never reach a chunk. The engine's +one primitive is an async stream of `(index, state, failure)` in chunk order, reordered through a ring of slots as +`ParallelCsvEnumerable` does; `AggregateAsync` folds the states with `Combine`, `ForEachAsync` ignores them, and +`ParseAsync` uses a `List` per chunk and yields its items. + +The XLSX row boundary is a candidate, not a proof: the same bytes can sit in CDATA, a comment or malformed +markup. So every chunk enumerator reports whether it ended cleanly, on a row boundary with no row or tag cut +short. A run of clean chunks equals the sequential read by induction. A chunk that does not end cleanly stops +before delivering its cut-short row; the merge keeps that chunk's rows, cancels the pipeline and reads the rest +with one sequential enumerator, skipping the rows already delivered, and later chunks' results are dropped. +This recovery re-inflates the sheet and only happens on such sheets. It leaves `ParseAsync` and `AggregateAsync` +exact, but `ForEachAsync` may already have called back for dropped chunks, so it is exactly once only for a +well-formed XLSX sheet. XLSB frames records from the first byte as the sequential reader does, so it has no +such case. + +The measured gain is bounded by inflate, which stays on one thread: see "Parallel intra-sheet parsing" in +[reader-performance.md](docs/performance/reader-performance.md). Parallel inflate would replace the producer +only; the queue, workers and merge stay. + +### The CSV engine + +`CsvSheet` takes the older path (`src/ExcelReader.Core/Parser/ParallelCsv/ParallelCsv*.cs`), over the same +typed-parsing machinery, for sources large enough that partitioning pays for itself. `ParallelCsvFactory` +decides eligibility, a seekable, UTF-8, large-enough source, and falls back to the ordinary sequential +`ExcelParser` otherwise, so callers get identical output either way. When it partitions, `CsvChunkPlan` slices the data range into four chunks per worker (not one — see the file's comments on why one-per-worker would buffer up to the whole output) and hands them out @@ -96,8 +156,8 @@ Still unresolved, and untested by any of the above: whether true unordered emiss differently. The ordered merge is the safer default and the only mode either public overload can reach, so it stays absent a specific reason to revisit. -`CsvParallel.ForEachAsync` exposes the same partitioning to a per-row callback rather than an -accumulator type. It accepts a `ref struct` row, which the enumerating `CsvParallel.ParseAsync` cannot — +`SheetParallel.ForEachAsync` exposes the same partitioning to a per-row callback rather than an +accumulator type. It accepts a `ref struct` row, which the enumerating `SheetParallel.ParseAsync` cannot — not because of the interface (`IAsyncEnumerable`'s `T` is declared `allows ref struct` on net9.0+), but because a row holding spans into a worker's buffer cannot cross to the consumer thread at all. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 888cc58c..82fea814 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,6 +32,14 @@ rewrites — see [ARCHITECTURE.md](ARCHITECTURE.md) for the shape of the codebas [STYLEGUIDE.md § Untrusted Input](STYLEGUIDE.md#untrusted-input) before touching a parser: every length, offset, and size read from the file must be bounded before it drives an allocation. +- **Guide examples compile.** A C# block in `docs/guide` marked `` is copied + from `#region name` in `docs/snippets`, which builds with the solution. Edit the region, then + rewrite the guides; CI fails if they differ: + + ```bash + python docs/snippets/sync.py + ``` + ## Commit messages Use [Conventional Commits](https://www.conventionalcommits.org/): `type(scope): summary`, imperative diff --git a/Directory.Build.props b/Directory.Build.props index f9b76a42..8782a287 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -27,7 +27,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/ExcelReader.slnx b/ExcelReader.slnx index cd54752f..fc0e09fe 100644 --- a/ExcelReader.slnx +++ b/ExcelReader.slnx @@ -13,4 +13,7 @@ + + + diff --git a/README.md b/README.md index 57376477..1d410915 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +![ExcelReader logo](https://raw.githubusercontent.com/GabrielMarquezMatte/ExcelReader/master/assets/icon.png) + # ExcelReader [![CI](https://github.com/GabrielMarquezMatte/ExcelReader/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/GabrielMarquezMatte/ExcelReader/actions/workflows/ci.yml) @@ -40,6 +42,8 @@ foreach (Row row in workbook.FirstSheet) - [Writing workbooks](docs/guide/writing.md) — XLSX, XLSB (BIFF12) and XLS (BIFF8) writers, cell styles, typed records, and prefetch compression. - [CSV](docs/guide/csv.md) — reading, dialect sniffing, parallel parsing, and writing delimited text. - [Encrypted workbooks](docs/guide/encryption.md) — opening password-protected packages and encrypting written ones. +- [Reading untrusted files](docs/guide/untrusted-input.md) — the resource limits that bound what one hostile file can cost, and how to reject one cleanly. +- [Migrating to v7](docs/guide/migrating-to-v7.md) — porting `CsvParallel` code to `SheetParallel`, which now reads XLSX and XLSB on several threads too. - [Migrating to v6](docs/guide/migrating-to-v6.md) — porting v5 reader code to workbooks and sheets, and reading sheets in parallel. - [Benchmarks](docs/performance/benchmarks.md) — throughput and allocation against Sep, Sylvan, OfficeIMO and SpreadCheetah. [Live results](https://gabrielmarquezmatte.github.io/ExcelReader/dev/bench/). @@ -75,7 +79,7 @@ errors to stderr, so `convert` is safe to pipe. - Reads `.xlsx`, `.xlsb` (BIFF12), `.xls` (BIFF8), and `.csv`; writes `.xlsx`, `.xlsb`, `.xls`, and `.csv`. - `Excel.Open`/`OpenAsync` auto-detect XLSX/XLSB/XLS from the file signature. CSV has none, so pass `ExcelFileFormat.Csv` to open delimited text through the same entry point; `ExcelReaderOptions.Csv` carries the dialect, and `CsvReaderOptions.SniffDialect` infers it from the source. - A workbook hands out its sheets: `FirstSheet`, `Sheets[index]` or `SheetAt(index)`, and `TryGetSheet(name)`. CSV is a workbook with one unnamed sheet. Coming from v5? See [Migrating to v6](docs/guide/migrating-to-v6.md). -- Sheets of one workbook can be read in parallel, one enumerator per thread. +- Sheets of one workbook can be read in parallel, one enumerator per thread. One sheet of any format (XLSX, XLSB or CSV) can also be read on several threads with `SheetParallel`. - `sheet.Visibility` reports whether a sheet is hidden (`Visible`, `Hidden`, `VeryHidden`). Hidden sheets still enumerate their rows — filter on the value if you want them skipped. The writers take the same value via `AddSheet(name, visibility)`, and reject a workbook whose sheets would all be hidden. - Missing cells in sparse rows are exposed as empty cells. - String conversion allocates only when you call `GetString()`. diff --git a/STYLEGUIDE.md b/STYLEGUIDE.md index 7cb2cb22..63275833 100644 --- a/STYLEGUIDE.md +++ b/STYLEGUIDE.md @@ -181,7 +181,7 @@ Never reference the task, the ticket, or the caller. ### Findings do not belong in code Measurements, benchmark results, memory-footprint numbers, and the reasoning that picked one -approach over another are **not comments**. They go in [`docs/`](docs/). +approach over another are **not comments**. They go in [`docs/`](docs/README.md). This includes the whole class of "why it is this way" content: a throughput figure that justified a buffer size, an allocation count that motivated pooling, a design that was prototyped and measured @@ -209,7 +209,7 @@ Comments carry maintenance cost. Two failure modes are specifically banned: - **Do not cite a file that can disappear.** Never point a comment at a plan, a spec, or a section marker ("see step Z4") as the authoritative explanation. Those are working artifacts: they get renamed, merged and deleted, and the comment then sends a reader to nothing, which is worse than - silence. A committed document under [`docs/`](docs/) is different in kind — it is permanent, and + silence. A committed document under [`docs/`](docs/README.md) is different in kind — it is permanent, and pointing at one is how a finding stays out of the code. The test is whether the target is expected to still exist in a year, not whether a link is involved. @@ -329,7 +329,7 @@ These rules apply only when a function is on a measured hot path (cell parsing, - Avoid LINQ in hot paths; use `IndexOf` on spans which benefits from SIMD. - Split cold paths (growth, throw helpers) into `[MethodImpl(MethodImplOptions.NoInlining)]` methods so they do not bloat the hot caller's IL and cost it inlining. -- Optimize against a measurement, and record what it showed in [`docs/`](docs/) — never in a comment. +- Optimize against a measurement, and record what it showed in [`docs/`](docs/README.md) — never in a comment. An optimization with no number behind it is a guess that costs readability. Caches in particular are a trade, not a win: document the workload where the cache pays and the one where it does not, with the numbers that showed it. diff --git a/THIRD-PARTY-NOTICES.md b/THIRD-PARTY-NOTICES.md index e9fc20f2..c67d0e57 100644 --- a/THIRD-PARTY-NOTICES.md +++ b/THIRD-PARTY-NOTICES.md @@ -1,7 +1,7 @@ # Third-Party Notices This project (ExcelReader.NET) is licensed under the MIT License (see `LICENSE`). It vendors one -piece of third-party test data, noted below. +piece of third-party test data and ports one piece of third-party code, both noted below. ## tests/ExcelReader.Tests/data/encrypted/standard-aes128-sha1.xlsx @@ -14,3 +14,35 @@ It is used solely as a third-party oracle fixture to verify ECMA-376 standard-en against a file this codebase did not produce. It is test-project content only: it ships in the `ExcelReader.Tests` test project, never in the `ExcelReader.Core` NuGet package (`PackageLicenseExpression=MIT`), and is not distributed in any published package. + +## src/ExcelReader.Core/Reader/Zip/Inflate/ + +The inflate decoder in this folder is a C# port of the decompressor design in libdeflate: its +decode-table layout, table-building algorithm and fast-loop structure. + +- Project: libdeflate — https://github.com/ebiggers/libdeflate +- License: MIT + +This code ships in the `ExcelReader.NET` NuGet package, so the notice below ships with it. + + Copyright 2016 Eric Biggers + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation files + (the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of the Software, + and to permit persons to whom the Software is furnished to do so, + subject to the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS + BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN + ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. diff --git a/assets/icon.png b/assets/icon.png new file mode 100644 index 00000000..72e431aa Binary files /dev/null and b/assets/icon.png differ diff --git a/assets/icon.svg b/assets/icon.svg new file mode 100644 index 00000000..bf83f09d --- /dev/null +++ b/assets/icon.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/assets/social-preview.png b/assets/social-preview.png new file mode 100644 index 00000000..33f1aa11 Binary files /dev/null and b/assets/social-preview.png differ diff --git a/cpp/README.md b/cpp/README.md index d21e5162..d756c881 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -1,3 +1,5 @@ +![ExcelReader logo](https://raw.githubusercontent.com/GabrielMarquezMatte/ExcelReader/master/assets/icon.png) + # excelreader (C++) Header-only C++23 wrapper around ExcelReader's native C ABI: opening a workbook (from a path or @@ -245,6 +247,33 @@ if (!encrypted) { /* encrypted.error().message */ } Encryption parameters are fixed at Excel's own defaults — there are no options — and only XLSX/XLSB packages can be encrypted, matching what `Workbook::open`/`open_memory` can decrypt. +### Reading from your own bytes + +Derive from `xl::Source` to read from anything you can read at an offset; `read_at` is called from +several threads, in cached 4 MiB blocks. Derive from `xl::InputStream` for a stream that cannot +seek: a CSV is read as it arrives, an XLSX, XLSB or XLS is read whole first. + +```cpp +class StdinStream final : public xl::InputStream +{ +public: + std::expected read(std::span buffer) override + { + std::cin.read(reinterpret_cast(buffer.data()), static_cast(buffer.size())); + if (std::cin.bad()) + { + return std::unexpected(std::string("reading stdin failed")); + } + return static_cast(std::cin.gcount()); + } +}; + +auto workbook = xl::Workbook::open_stream(std::make_unique(), XL_FORMAT_CSV); +``` + +The workbook owns the object you pass and deletes it once nothing needs it. A returned error +string or a thrown exception becomes the `xl::Error` of the call that needed the bytes. + ### Arrow export `` is a separate header — including `` never pulls the @@ -341,15 +370,15 @@ on Windows 10 (22H2), 16 logical CPUs @ 3.39 GHz, MSVC 19.51 (Release), Google B | Benchmark | RealExcel.xlsb (100 rows) | 65K_Records_Data.xlsb (65,535 rows) | |---|---:|---:| -| `open` (`open_memory`) | 20.0 µs | 1.24 ms | -| `parse_sheet` (4 or 6 bound columns) | 50.3 µs | 27.3 ms | -| `typed_reader`, batches of 1,000 / 10,000 rows | - | 26.1 ms / 26.5 ms | -| `infer_schema` (sample 100 / 1,000 rows) | 73.7 µs | 605.7 µs | +| `open` (`open_memory`) | 24.0 µs | 1.17 ms | +| `parse_sheet` (4 or 6 bound columns) | 47.8 µs | 24.7 ms | +| `typed_reader`, batches of 1,000 / 10,000 rows | - | 23.8 ms / 23.8 ms | +| `infer_schema` (sample 100 / 1,000 rows) | 75.6 µs | 579.1 µs | `open` here is `open_memory`, which copies the caller's buffer before returning (the caller may free -it afterwards), so it scales with file size: 1.24 ms is mostly copying the 3.7 MB fixture. Opening by +it afterwards), so it scales with file size: 1.17 ms is mostly copying the 3.7 MB fixture. Opening by path reads only the package directory and stays nearly flat — see the Rust suite below. -`parse_sheet` scales linearly with rows × columns, at roughly 420 ns/row here. Draining the same +`parse_sheet` scales linearly with rows × columns, at roughly 380 ns/row here. Draining the same sheet through the batched typed reader costs the same as one whole-sheet `parse_sheet` while holding only one batch in memory at a time. `infer_schema` scales with its sample size, not the file's total row count. @@ -364,12 +393,12 @@ sides decode every cell into an owned value (`std::string` for text, not the zer | Library | Mean | |---|---:| -| ExcelReader (`parse_sheet`) | 73.7 ms | -| DuckDB (`read_xlsx`, summing every column in SQL) | 433.6 ms | -| xlsxio (`xlsxioread_sheet_next_cell_*`) | 505.0 ms | -| xlnt (`worksheet::rows()` + `cell::to_string()`/`value()`) | 2,494.2 ms | +| ExcelReader (`parse_sheet`) | 62.9 ms | +| DuckDB (`read_xlsx`, summing every column in SQL) | 410.8 ms | +| xlsxio (`xlsxioread_sheet_next_cell_*`) | 512.9 ms | +| xlnt (`worksheet::rows()` + `cell::to_string()`/`value()`) | 2,394.1 ms | -ExcelReader is ~5.9x faster than DuckDB, ~6.9x faster than xlsxio and ~34x faster than xlnt on this +ExcelReader is ~6.5x faster than DuckDB, ~8.2x faster than xlsxio and ~38x faster than xlnt on this workload. xlsxio is a lean, purpose-built C streaming reader — the same abstraction level as ExcelReader's own native core — so it was the strongest of the two competitors tested, though still well behind. @@ -409,11 +438,11 @@ Measured on the same machine as above, GCC 16.2.0 (MSYS2 UCRT64, `-O3 -mavx2` fo | Workload | ExcelReader | zsv `compat` | zsv `fast` | |---|---:|---:|---:| -| Typed, 14 columns into `FullRow` | 30.4 ms | 25.7 ms | 22.7 ms | -| Cells, byte count | 11.9 ms | 5.9 ms | 3.5 ms | +| Typed, 14 columns into `FullRow` | 27.2 ms | 23.4 ms | 20.4 ms | +| Cells, byte count | 11.8 ms | 6.5 ms | 3.6 ms | **zsv is faster on both.** Typed, it is ~1.2x (`compat`) and ~1.3x (`fast`) faster than ExcelReader. -Counting cells, it is ~2.0x and ~3.4x faster. +Counting cells, it is ~1.8x and ~3.3x faster. The cells gap is the C ABI, not the CSV parser. `xl_rows_next_view` copies every cell's value into a native row the cursor owns, one call per row, and that alone costs ~10.6 ms here (calling it without @@ -450,11 +479,11 @@ is handed buffers that are already columnar, while `BM_WriteSheet` starts from a | Benchmark | Time | Rows/s | |---|---:|---:| -| `BM_WriteColumns` (pre-transposed) | 44.2 ms | 1.48 M/s | -| `BM_WriteSheet` (from `std::vector`) | 50.9 ms | 1.29 M/s | +| `BM_WriteColumns` (pre-transposed) | 45.8 ms | 1.43 M/s | +| `BM_WriteSheet` (from `std::vector`) | 52.2 ms | 1.27 M/s | -The transpose costs ~15% here. It is not free, but it is not the dominant cost of producing the -file — see the comparison below, where the same two cases over 14 columns land within ~15–17%. +The transpose costs ~14% here. It is not free, but it is not the dominant cost of producing the +file — see the comparison below, where the same two cases over 14 columns land within ~12%. `excelreader_cpp_write_compare_benchmarks` (under `-DEXCELREADER_BUILD_BENCHMARKS_COMPARE=ON`) puts that against xlnt, xlsxio and [DuckDB](https://github.com/duckdb/duckdb)'s `excel` extension, all @@ -471,12 +500,12 @@ competitor. | Library | Wall | CPU | |---|---:|---:| -| ExcelReader (`xl::write_columns`, pre-transposed) | 85.1–85.9 ms | 85.1–87.1 ms | -| ExcelReader (`xl::write_sheet`) | 97.8–100.1 ms | 96.4–100.4 ms | -| DuckDB (`COPY ... TO ... WITH (FORMAT xlsx)`) | 833.3–869.5 ms | 828.1–875.0 ms | -| libxlsxwriter (`worksheet_write_string`/`_number`) | 1,183.8 ms | 1,187.5 ms | -| xlsxio (`xlsxiowrite_add_cell_*`) | 2,453.8 ms | 843.8 ms | -| xlnt (`worksheet::cell().value()` + `save()`) | 5,640.7 ms | 5,640.6 ms | +| ExcelReader (`xl::write_columns`, pre-transposed) | 85.4–86.5 ms | 85.1–87.1 ms | +| ExcelReader (`xl::write_sheet`) | 95.5–96.5 ms | 96.0–98.2 ms | +| DuckDB (`COPY ... TO ... WITH (FORMAT xlsx)`) | 828.9–833.2 ms | 828.1 ms | +| libxlsxwriter (`worksheet_write_string`/`_number`) | 1,193.9 ms | 1,203.1 ms | +| xlsxio (`xlsxiowrite_add_cell_*`) | 2,433.2 ms | 953.1 ms | +| xlnt (`worksheet::cell().value()` + `save()`) | 5,436.3 ms | 5,437.5 ms | Same machine as above; Google Benchmark's own iteration counts, no `--benchmark_repetitions` (each iteration writes a whole 65,535-row file, so the slower cases run once or a handful of times). xlnt @@ -486,8 +515,8 @@ DuckDB rows appear in both, and the small ranges above are those two independent sampling within one run. `write_sheet` — the matched-work number, since it starts from the same `std::vector` every -competitor is handed — is ~8.5–8.7x faster than DuckDB, ~12.1x faster than libxlsxwriter, ~25x -faster than xlsxio, and ~56x faster than xlnt on wall time. +competitor is handed — is ~8.6–8.7x faster than DuckDB, ~12.4x faster than libxlsxwriter, ~25x +faster than xlsxio, and ~57x faster than xlnt on wall time. ### Known issue: xlsxio + libxlsxwriter cannot share a binary @@ -524,10 +553,10 @@ Caveats, none of them optional when quoting these: - **xlsxio's CPU time is not stable across runs.** It read 843.8–906.3 ms before the fix (some runs missing the workbook.xml entry), 1,171.9 ms and 687.5 ms in the two runs after it, - 1,015.6 ms in the next and 843.8 ms in this one, while its wall time stayed at ~2.4 s throughout. Its wall time is dominated by I/O wait — + 1,015.6 ms and 843.8 ms in the next two and 953.1 ms in the latest, while its wall time stayed at ~2.4 s throughout. Its wall time is dominated by I/O wait — CPU is well under half of wall, against every other case here being CPU-bound (wall ≈ CPU) — so quote the wall-time ratio, which is what a caller waits. Against this run's CPU time the gap from - `write_sheet` to xlsxio is ~8.4x, not ~25x. + `write_sheet` to xlsxio is ~9.9x, not ~25x. - **ExcelReader does slightly more work here**, not less: it attaches a number format to the two date columns so Excel shows a date, while every competitor case writes those as bare serial numbers. That difference favours the competitors. diff --git a/cpp/benchmarks/benchmark_csv_compare.cpp b/cpp/benchmarks/benchmark_csv_compare.cpp index cb0eb531..211faf13 100644 --- a/cpp/benchmarks/benchmark_csv_compare.cpp +++ b/cpp/benchmarks/benchmark_csv_compare.cpp @@ -1,8 +1,6 @@ #include #include -// zsv's headers are plain C99: no extern "C" guard, and `restrict`, which GCC and Clang spell -// `__restrict` in C++. #define restrict __restrict extern "C" { @@ -135,8 +133,6 @@ namespace return {reinterpret_cast(cell.str), cell.len}; } - // The same conversions parse_sheet makes: text into owned std::string, ISO dates, integers - // and doubles, one FullRow per data row, header skipped. void zsv_typed_row(void *context) { auto *run = static_cast(context); @@ -146,8 +142,6 @@ namespace return; } const size_t count = zsv_cell_count(run->parser); - // zsv reports the empty line after the file's final CRLF as a one-cell row; ExcelReader skips - // blank lines. if (count == 1 && zsv_get_cell(run->parser, 0).len == 0) { return; diff --git a/cpp/scripts/api_markdown.py b/cpp/scripts/api_markdown.py new file mode 100644 index 00000000..f5ac9f37 --- /dev/null +++ b/cpp/scripts/api_markdown.py @@ -0,0 +1,365 @@ +"""Write the C++ API reference as Markdown pages plus a toc.yml, for docfx to render. + +Runs Doxygen over the public headers and reads its XML output, so Doxygen must be on PATH. + +Run from anywhere: python cpp/scripts/api_markdown.py +""" + +import re +import subprocess +import tempfile +import textwrap +import xml.etree.ElementTree as ET +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +HEADERS = ["cpp/include/xl/excelreader.hpp", "cpp/include/xl/excelreader_arrow.hpp"] +OUTPUT_DIR = REPO_ROOT / "docs" / "cpp-api" +MAX_LINE = 100 + + +def run_doxygen(output: Path) -> Path: + config = "\n".join( + [ + f"INPUT = {' '.join(HEADERS)}", + f"OUTPUT_DIRECTORY = {output.as_posix()}", + "GENERATE_HTML = NO", + "GENERATE_LATEX = NO", + "GENERATE_XML = YES", + "XML_PROGRAMLISTING = NO", + "EXTRACT_ALL = YES", + "EXCLUDE_SYMBOLS = xl::detail*", + "QUIET = YES", + "WARN_IF_UNDOCUMENTED = NO", + "WARN_IF_DOC_ERROR = YES", + "WARN_IF_INCOMPLETE_DOC = YES", + "WARN_NO_PARAMDOC = YES", + "WARN_AS_ERROR = FAIL_ON_WARNINGS", + ] + ) + if subprocess.run(["doxygen", "-"], input=config, text=True, cwd=REPO_ROOT).returncode != 0: + raise SystemExit("Doxygen reported documentation problems in the headers; see its warnings above.") + return output / "xml" + + +def tidy(text: str) -> str: + text = re.sub(r"\s*<\s*", "<", text) + text = re.sub(r"\s+>", ">", text) + return re.sub(r"\s+", " ", text).strip() + + +def flat(element) -> str: + return tidy("".join(element.itertext())) if element is not None else "" + + +SECTIONS = ("parameterlist", "simplesect") + + +def inline(node) -> str: + """A paragraph as Markdown text: inline code kept, @param/@return blocks left to their own readers.""" + out = node.text or "" + for child in node: + if child.tag not in SECTIONS: + inner = inline(child) + out += f"`{inner}`" if child.tag == "computeroutput" else inner + out += child.tail or "" + return re.sub(r"\s+", " ", out).strip() + + +def prose(element) -> str: + if element is None: + return "" + paragraphs = [inline(p) for p in element.findall("para")] + notes = [ + f"**{s.get('kind', '').capitalize()}** " + " ".join(inline(p) for p in s.findall("para")) + for p in element.findall("para") + for s in p.findall("simplesect") + if s.get("kind") not in ("return", "returns") + ] + return "\n\n".join(p for p in paragraphs + notes if p) + + +def description(node) -> str: + return "\n\n".join(p for p in (prose(node.find("briefdescription")), prose(node.find("detaileddescription"))) if p) + + +def documented_params(node, kind: str) -> dict[str, str]: + """The `@param` (kind "param") or `@tparam` (kind "templateparam") texts, by name.""" + found: dict[str, str] = {} + if node is None: + return found + for parameter_list in node.iter("parameterlist"): + if parameter_list.get("kind") != kind: + continue + for item in parameter_list.findall("parameteritem"): + text = " ".join(inline(p) for p in item.iter("para")) + for name in item.iter("parametername"): + found.setdefault(flat(name), text) + return found + + +def documented_return(node) -> str: + if node is None: + return "" + return " ".join( + inline(p) for s in node.iter("simplesect") if s.get("kind") in ("return", "returns") for p in s.findall("para") + ) + + +def type_parameters(node) -> str: + docs = documented_params(node, "templateparam") + if not docs: + return "" + rows = "\n".join(f"| `{name}` | {cell(text)} |" for name, text in docs.items()) + return f"**Type parameters**\n\n| Name | Description |\n|---|---|\n{rows}" + + +def cell(text: str) -> str: + return text.replace("|", "\\|") + + +def linked(text: str, linkable: dict[str, str]) -> str: + text = cell(text).replace("<", "<").replace(">", ">") + return re.sub( + r"(? str: + return f"```cpp\n{text}\n```" + + +def template_prefix(node) -> str: + params = node.find("templateparamlist") + if params is None: + return "" + names = [] + for param in params.iter("param"): + declared = flat(param.find("type")) + name = param.findtext("declname") or param.findtext("defname") or "" + names.append(tidy(f"{declared} {name}")) + return f"template <{', '.join(names)}>\n" + + +def signature(member) -> str: + name = member.findtext("name") + returns = flat(member.find("type")) + prefix = "static " if member.get("static") == "yes" else "" + head = tidy(f"{prefix}{returns} {name}") + args = tidy(member.findtext("argsstring") or "") + one_line = f"{template_prefix(member)}{head}{args}" + params = member.findall("param") + if len(one_line.splitlines()[-1]) <= MAX_LINE or not params: + return one_line + + lines = [] + for param in params: + text = tidy(f"{flat(param.find('type'))} {param.findtext('declname') or ''}") + default = flat(param.find("defval")) + lines.append(f"{text} = {default}" if default else text) + trailer = args[args.rindex(")") + 1 :].strip() + body = ",\n".join(f" {line}" for line in lines) + return f"{template_prefix(member)}{head}(\n{body}){(' ' + trailer) if trailer else ''}" + + +def parameters_and_returns(members: list, linkable: dict[str, str]) -> str: + parts = [type_parameters(members[0])] + docs: dict[str, str] = {} + for member in members: + for name, text in documented_params(member, "param").items(): + docs.setdefault(name, text) + + seen = set() + rows = [] + for param in (p for m in members for p in m.findall("param") if p.findtext("declname")): + name = param.findtext("declname") + type_text = flat(param.find("type")) + if (name, type_text) in seen: + continue + seen.add((name, type_text)) + default = flat(param.find("defval")) + rows.append( + f"| `{name}` | {linked(type_text, linkable)} | " + + (f"`{cell(default)}`" if default else "") + + (f" | {cell(docs.get(name or '', ''))}" if docs else "") + + " |" + ) + if rows: + header = "| Name | Type | Default | Description |\n|---|---|---|---|" if docs else "| Name | Type | Default |\n|---|---|---|" + parts.append(f"**Parameters**\n\n{header}\n" + "\n".join(rows)) + + returns = {re.sub(r"\b(constexpr|inline|static|explicit) ", "", flat(m.find("type"))) for m in members} + returns -= {"", "void"} + if returns: + text = next((documented_return(m) for m in members if documented_return(m)), "") + parts.append( + "**Returns** " + ", ".join(linked(r, linkable) for r in sorted(returns)) + (f": {text}" if text else "") + ) + return "\n\n".join(p for p in parts if p) + + +def by_name(members: list) -> dict[str, list]: + grouped: dict[str, list] = {} + for member in members: + grouped.setdefault(member.findtext("name"), []).append(member) + return grouped + + +def member_sections(title: str, members: list, linkable: dict[str, str], level: str) -> list[str]: + if not members: + return [] + parts = [f"{level} {title}"] + for name, overloads in by_name(members).items(): + parts += [ + f"{level}# {name}", + code("\n".join(signature(m) for m in overloads)), + *dict.fromkeys(description(m) for m in overloads), + parameters_and_returns(overloads, linkable), + ] + return parts + + +def is_copy_or_move(member, class_name: str) -> bool: + params = member.findall("param") + return ( + member.findtext("name") in (class_name, "operator=") + and len(params) == 1 + and re.fullmatch(rf"(const )?{re.escape(class_name)} ?&&?", flat(params[0].find("type"))) is not None + ) + + +def class_page(compound, linkable: dict[str, str]) -> str: + full_name = compound.findtext("compoundname") + short = full_name.removeprefix("xl::") + kind = compound.get("kind") + parts = [ + f"# {kind.capitalize()} {short}", + description(compound), + code(f"{template_prefix(compound)}{kind} {full_name}"), + type_parameters(compound.find("detaileddescription")), + ] + + public = [ + m + for m in compound.iter("memberdef") + if m.get("prot") == "public" and "=delete" not in (m.findtext("argsstring") or "").replace(" ", "") + ] + fields = [m for m in public if m.get("kind") == "variable"] + functions = [ + m + for m in public + if m.get("kind") == "function" and not m.findtext("name").startswith("~") and not is_copy_or_move(m, short) + ] + constructors = [m for m in functions if m.findtext("name") == short.split("<")[0]] + operators = [m for m in functions if m.findtext("name").startswith("operator")] + methods = [m for m in functions if m not in constructors and m not in operators] + + if fields: + described = any(description(m) for m in fields) + rows = "\n".join( + f"| `{m.findtext('name')}` | {linked(flat(m.find('type')), linkable)} | " + + (f"`{cell(flat(m.find('initializer')).lstrip('= '))}`" if m.find("initializer") is not None else "") + + (f" | {cell(description(m)).replace(chr(10), ' ')}" if described else "") + + " |" + for m in fields + ) + header = "| Name | Type | Default | Description |\n|---|---|---|---|" if described else "| Name | Type | Default |\n|---|---|---|" + parts.append(f"## Fields\n\n{header}\n{rows}") + + parts += member_sections("Constructors", constructors, linkable, "##") + parts += member_sections("Methods", methods, linkable, "##") + parts += member_sections("Operators", operators, linkable, "##") + return "\n\n".join(p for p in parts if p) + "\n" + + +def enum_page(member) -> str: + name = member.findtext("name") + underlying = flat(member.find("type")) + rows = "\n".join( + f"| `{v.findtext('name')}` | `{cell(flat(v.find('initializer')).lstrip('= '))}` | {cell(description(v))} |" + for v in member.iter("enumvalue") + ) + parts = [ + f"# Enum {name}", + description(member), + code(f"enum class xl::{name}" + (f" : {underlying}" if underlying else "")), + f"## Members\n\n| Name | Value | Description |\n|---|---|---|\n{rows}", + ] + return "\n\n".join(p for p in parts if p) + "\n" + + +def concept_page(concept) -> str: + name = concept.findtext("compoundname", "").removeprefix("xl::") + initializer = concept.find("initializer") + definition = textwrap.dedent("".join(initializer.itertext())).strip() if initializer is not None else "" + parts = [f"# Concept {name}", description(concept), code(definition), type_parameters(concept.find("detaileddescription"))] + return "\n\n".join(p for p in parts if p) + "\n" + + +def functions_page(functions: list, linkable: dict[str, str]) -> str: + parts = ["# Functions", "Free functions in namespace `xl`."] + for name, overloads in by_name(functions).items(): + parts += [ + f"## {name}", + code("\n".join(signature(m) for m in overloads)), + *dict.fromkeys(description(m) for m in overloads), + parameters_and_returns(overloads, linkable), + ] + return "\n\n".join(p for p in parts if p) + "\n" + + +def toc_section(title: str, names: list[str]) -> str: + items = "".join(f" - name: {n}\n href: {n}.md\n" for n in names) + return f"- name: {title}\n items:\n{items}" if names else "" + + +def main() -> None: + with tempfile.TemporaryDirectory() as scratch: + xml_dir = run_doxygen(Path(scratch)) + compounds = [] + concepts = [ET.parse(path).getroot()[0] for path in sorted(xml_dir.glob("concept*.xml"))] + for path in sorted(xml_dir.glob("*.xml")): + if path.name.startswith(("class", "struct")): + compound = ET.parse(path).getroot()[0] + name = compound.findtext("compoundname", "") + if name.count("::") == 1 and "<" not in name: + compounds.append(compound) + namespace = ET.parse(xml_dir / "namespacexl.xml").getroot()[0] + + members = list(namespace.iter("memberdef")) + enums = [m for m in members if m.get("kind") == "enum"] + functions = [m for m in members if m.get("kind") == "function"] + + pages = {c.findtext("compoundname", "").removeprefix("xl::"): c for c in compounds} + linkable = {name: f"{name}.md" for name in pages} + linkable.update({e.findtext("name", ""): f"{e.findtext('name')}.md" for e in enums}) + concept_names = [c.findtext("compoundname", "").removeprefix("xl::") for c in concepts] + linkable.update({name: f"{name}.md" for name in concept_names}) + + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + for stale in OUTPUT_DIR.iterdir(): + stale.unlink() + for name, compound in pages.items(): + (OUTPUT_DIR / f"{name}.md").write_text(class_page(compound, linkable), encoding="utf-8") + for enum in enums: + (OUTPUT_DIR / f"{enum.findtext('name')}.md").write_text(enum_page(enum), encoding="utf-8") + for name, concept in zip(concept_names, concepts): + (OUTPUT_DIR / f"{name}.md").write_text(concept_page(concept), encoding="utf-8") + (OUTPUT_DIR / "functions.md").write_text(functions_page(functions, linkable), encoding="utf-8") + + classes = sorted(n for n, c in pages.items() if c.get("kind") == "class") + structs = sorted(n for n, c in pages.items() if c.get("kind") == "struct") + toc = ( + toc_section("Classes", classes) + + toc_section("Structs", structs) + + toc_section("Enums", sorted(e.findtext("name", "") for e in enums)) + + toc_section("Concepts", sorted(concept_names)) + + "- name: Functions\n href: functions.md\n" + ) + (OUTPUT_DIR / "toc.yml").write_text(toc, encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt index 3e166f8a..a91732ae 100644 --- a/cpp/tests/CMakeLists.txt +++ b/cpp/tests/CMakeLists.txt @@ -38,6 +38,15 @@ target_compile_definitions(excelreader_cpp_rows PRIVATE add_test(NAME excelreader_cpp_rows COMMAND excelreader_cpp_rows) excelreader_copy_native_library(excelreader_cpp_rows) +add_executable(excelreader_cpp_sources sources.cpp) +find_package(Threads REQUIRED) +target_link_libraries(excelreader_cpp_sources PRIVATE xl::excelreader Threads::Threads) +target_compile_definitions(excelreader_cpp_sources PRIVATE + EXCELREADER_XLSX_FIXTURE_PATH="${EXCELREADER_XLSX_FIXTURE_PATH}") + +add_test(NAME excelreader_cpp_sources COMMAND excelreader_cpp_sources) +excelreader_copy_native_library(excelreader_cpp_sources) + add_executable(excelreader_cpp_sheets sheets.cpp) find_package(Threads REQUIRED) target_link_libraries(excelreader_cpp_sheets PRIVATE xl::excelreader Threads::Threads) diff --git a/cpp/tests/sources.cpp b/cpp/tests/sources.cpp new file mode 100644 index 00000000..a84afe35 --- /dev/null +++ b/cpp/tests/sources.cpp @@ -0,0 +1,277 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace +{ + int failures = 0; + + void check(bool condition, const char *what) + { + if (!condition) + { + std::fprintf(stderr, "FAILED: %s\n", what); + ++failures; + } + } + + std::vector read_file(const char *path) + { + std::ifstream in(path, std::ios::binary); + std::vector chars((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + std::vector bytes(chars.size()); + std::memcpy(bytes.data(), chars.data(), chars.size()); + return bytes; + } + + struct Counters + { + std::atomic reads{0}; + std::atomic destroyed{0}; + }; + + class MemorySource final : public xl::Source + { + public: + MemorySource(std::vector bytes, Counters &counters) : bytes_(std::move(bytes)), counters_(counters) {} + ~MemorySource() override { counters_.destroyed.fetch_add(1); } + + std::uint64_t size() const override { return bytes_.size(); } + + std::expected read_at(std::uint64_t offset, std::span buffer) const override + { + counters_.reads.fetch_add(1); + if (offset >= bytes_.size()) + { + return std::size_t{0}; + } + const std::size_t count = std::min(buffer.size(), bytes_.size() - offset); + std::memcpy(buffer.data(), bytes_.data() + offset, count); + return count; + } + + private: + std::vector bytes_; + Counters &counters_; + }; + + class FailingSource final : public xl::Source + { + public: + explicit FailingSource(Counters &counters) : counters_(counters) {} + ~FailingSource() override { counters_.destroyed.fetch_add(1); } + + std::uint64_t size() const override { return 1024; } + std::expected read_at(std::uint64_t, std::span) const override + { + return std::unexpected(std::string("disk on fire")); + } + + private: + Counters &counters_; + }; + + class ThrowingSource final : public xl::Source + { + public: + explicit ThrowingSource(Counters &counters) : counters_(counters) {} + ~ThrowingSource() override { counters_.destroyed.fetch_add(1); } + + std::uint64_t size() const override { return 1024; } + std::expected read_at(std::uint64_t, std::span) const override + { + throw std::runtime_error("boom"); + } + + private: + Counters &counters_; + }; + + class ErrorStream final : public xl::InputStream + { + public: + std::expected read(std::span) override + { + return std::unexpected(std::string("pipe broke")); + } + }; + + class ChunkStream final : public xl::InputStream + { + public: + explicit ChunkStream(std::vector bytes) : bytes_(std::move(bytes)) {} + + std::expected read(std::span buffer) override + { + const std::size_t count = std::min({buffer.size(), std::size_t{7}, bytes_.size() - position_}); + std::memcpy(buffer.data(), bytes_.data() + position_, count); + position_ += count; + return count; + } + + private: + std::vector bytes_; + std::size_t position_ = 0; + }; + + std::vector build_sheets(int sheet_count, int row_count) + { + auto writer = xl::WriterHandle::open_memory(XL_FORMAT_XLSX); + std::vector bytes; + if (!writer.has_value()) + { + return bytes; + } + for (int s = 0; s < sheet_count; ++s) + { + (void)writer->start_sheet("sheet" + std::to_string(s)); + for (int r = 0; r < row_count; ++r) + { + (void)writer->start_row(); + (void)writer->write("s" + std::to_string(s) + "-r" + std::to_string(r)); + (void)writer->end_row(); + } + (void)writer->end_sheet(); + } + auto written = writer->bytes(); + if (written.has_value()) + { + bytes.resize(written->size()); + std::memcpy(bytes.data(), written->data(), written->size()); + } + return bytes; + } + + std::vector first_column(const xl::Workbook &workbook, int32_t sheet) + { + std::vector values; + auto cursor = workbook.sheet(sheet).rows(); + while (cursor.has_value()) + { + auto row = cursor->next_row(); + if (!row.has_value()) + { + break; + } + values.emplace_back((*row)[0].value); + } + return values; + } + + std::size_t row_count(const xl::Workbook &workbook) + { + auto decoded = workbook.sheet(0).read_all_decoded(); + return decoded.has_value() ? decoded->size() : 0; + } +} + +int main() +{ + const std::vector bytes = read_file(EXCELREADER_XLSX_FIXTURE_PATH); + auto reference = xl::Workbook::open(EXCELREADER_XLSX_FIXTURE_PATH); + check(reference.has_value(), "open the fixture by path"); + const std::size_t expected_rows = reference.has_value() ? row_count(*reference) : 0; + check(expected_rows > 1, "the fixture has rows"); + + Counters counters; + { + auto workbook = xl::Workbook::open_source(std::make_unique(bytes, counters)); + check(workbook.has_value(), "open_source opens the fixture"); + check(workbook.has_value() && row_count(*workbook) == expected_rows, "open_source reads every row"); + } + check(counters.destroyed.load() == 1, "the source is destroyed exactly once"); + + const int blocks = static_cast((bytes.size() + (4u << 20) - 1) / (4u << 20)); + check(counters.reads.load() <= blocks, "the default cache fetches each block once"); + + Counters uncached; + { + xl::OpenOptions options; + options.source_block_size = -1; + auto workbook = xl::Workbook::open_source(std::make_unique(bytes, uncached), XL_FORMAT_XLSX, &options); + check(workbook.has_value() && row_count(*workbook) == expected_rows, "open_source without the cache reads every row"); + } + check(uncached.reads.load() > blocks, "without the cache every read reaches read_at"); + + { + Counters failing; + auto workbook = xl::Workbook::open_source(std::make_unique(failing), XL_FORMAT_XLSX); + check(!workbook.has_value() && workbook.error().message.find("disk on fire") != std::string::npos, + "the source's error message reaches the caller"); + check(failing.destroyed.load() == 1, "a source whose read failed is destroyed exactly once"); + } + { + Counters throwing; + auto workbook = xl::Workbook::open_source(std::make_unique(throwing), XL_FORMAT_XLSX); + check(!workbook.has_value() && workbook.error().message.find("boom") != std::string::npos, + "an exception from the source becomes an error"); + check(throwing.destroyed.load() == 1, "a source that threw is destroyed exactly once"); + } + { + auto workbook = xl::Workbook::open_stream(std::make_unique(), XL_FORMAT_XLSX); + check(!workbook.has_value() && workbook.error().message.find("pipe broke") != std::string::npos, + "the stream's error message reaches the caller"); + } + { + constexpr int sheet_count = 4; + const std::vector many = build_sheets(sheet_count, 300); + check(!many.empty(), "build the multi-sheet workbook"); + Counters parallel_counters; + auto reference_many = xl::Workbook::open_memory(std::span(reinterpret_cast(many.data()), many.size())); + auto workbook = xl::Workbook::open_source(std::make_unique(many, parallel_counters), XL_FORMAT_XLSX); + check(reference_many.has_value() && workbook.has_value(), "open the multi-sheet workbook both ways"); + if (reference_many.has_value() && workbook.has_value()) + { + std::vector> parallel(sheet_count * 2); + { + std::vector threads; + for (std::size_t i = 0; i < parallel.size(); ++i) + { + threads.emplace_back([¶llel, &workbook, i] + { parallel[i] = first_column(*workbook, static_cast(i % sheet_count)); }); + } + for (std::thread &thread : threads) + { + thread.join(); + } + } + bool same = true; + for (std::size_t i = 0; i < parallel.size(); ++i) + { + const auto expected = first_column(*reference_many, static_cast(i % sheet_count)); + same = same && expected.size() == 300 && parallel[i] == expected; + } + check(same, "every sheet of a source read on its own thread matches the in-memory read"); + } + } + { + const std::string csv = "name,qty\nwidget,7\n"; + std::vector csv_bytes(csv.size()); + std::memcpy(csv_bytes.data(), csv.data(), csv.size()); + auto workbook = xl::Workbook::open_stream(std::make_unique(csv_bytes), XL_FORMAT_CSV); + check(workbook.has_value() && row_count(*workbook) == 2, "open_stream reads a CSV"); + } + { + auto workbook = xl::Workbook::open_stream(std::make_unique(bytes)); + check(workbook.has_value() && row_count(*workbook) == expected_rows, "open_stream reads an XLSX"); + } + + if (failures != 0) + { + std::fprintf(stderr, "%d check(s) failed\n", failures); + return EXIT_FAILURE; + } + std::puts("sources: OK"); + return EXIT_SUCCESS; +} diff --git a/docfx.json b/docfx.json new file mode 100644 index 00000000..5589a580 --- /dev/null +++ b/docfx.json @@ -0,0 +1,53 @@ +{ + "$schema": "https://raw.githubusercontent.com/dotnet/docfx/main/schemas/docfx.schema.json", + "metadata": [ + { + "src": [ + { + "src": "src", + "files": [ + "ExcelReader.Core/ExcelReader.Core.csproj", + "ExcelReader.Arrow/ExcelReader.Arrow.csproj" + ] + } + ], + "output": "docs/api" + } + ], + "build": { + "content": [ + { "files": [ "index.md", "toc.yml", "api/**.{yml,md}", "python-api/**.{yml,md}", "cpp-api/**.{yml,md}" ], "src": "docs" }, + { + "files": [ + "docs/README.md", + "docs/guide/**.md", + "docs/performance/**.md", + "python/README.md", + "rust/excelreader/README.md", + "cpp/README.md", + "ARCHITECTURE.md", + "CONTRIBUTING.md", + "CODE_OF_CONDUCT.md", + "SECURITY.md", + "STYLEGUIDE.md" + ] + } + ], + "resource": [ + { "files": [ "assets/icon.png", "LICENSE", "src/ExcelReader.Native/include/excelreader.h" ] }, + { "files": [ "404.html" ], "src": "docs" } + ], + "sitemap": { + "baseUrl": "https://gabrielmarquezmatte.github.io/ExcelReader/", + "changefreq": "weekly" + }, + "output": "docs/_site", + "template": [ "default", "modern", "docs/template" ], + "globalMetadata": { + "_appName": "ExcelReader", + "_appTitle": "ExcelReader", + "_appLogoPath": "assets/icon.png", + "_enableSearch": true + } + } +} diff --git a/docs/404.html b/docs/404.html new file mode 100644 index 00000000..85ed8539 --- /dev/null +++ b/docs/404.html @@ -0,0 +1,23 @@ + + + + + + + Page not found | ExcelReader + + + +

Page not found

+

This page does not exist, or it moved when the documentation was reorganized.

+ + + diff --git a/docs/README.md b/docs/README.md index 7385ab95..ce2642d9 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,7 +14,7 @@ changes. A document with the numbers, the machine and the corpus can be reproduc valuable documents here** — they are what stops the same idea being rebuilt next year. - A performance ceiling and what it is bounded by. -- Usage documentation under [`guide/`](guide/), split out of the root README so the NuGet landing +- Usage documentation under [`guide/`](#index), split out of the root README so the NuGet landing page stays short. This is prose for a caller deciding *how* to use the library, not a substitute for the `///` comments that document *what* each member does. @@ -42,6 +42,7 @@ invariant) without restating the evidence for it. each format, the ceiling on each, and the optimizations measured and rejected. - [`performance/benchmarks.md`](performance/benchmarks.md) — throughput and allocation against Sep, Sylvan, OfficeIMO and SpreadCheetah. -- [`guide/`](guide/) — usage documentation: [reading](guide/reading.md), - [parsing](guide/parsing.md), [writing](guide/writing.md), [csv](guide/csv.md), - [encryption](guide/encryption.md). +- `guide/` — usage documentation: [reading](./guide/reading.md), + [parsing](./guide/parsing.md), [writing](./guide/writing.md), [csv](./guide/csv.md), + [encryption](./guide/encryption.md), [errors](./guide/errors.md), + [untrusted input](./guide/untrusted-input.md). diff --git a/docs/guide/csv.md b/docs/guide/csv.md index 074acf38..f62ca67d 100644 --- a/docs/guide/csv.md +++ b/docs/guide/csv.md @@ -1,12 +1,13 @@ # CSV Reading, dialect sniffing and writing delimited text. -Parallel CSV parsing is covered in [parsing.md](parsing.md). +Parallel CSV parsing is covered in [reading.md](reading.md#parallel-reading-opt-in). ## Read CSV -`CsvReader` is a workbook with exactly one, unnamed sheet (`csv.FirstSheet`, the same sheet as `csv.Sheets[0]`). It streams RFC 4180 CSV (quoted fields, embedded delimiters/newlines, `""`-escaped quotes) through the same `Row`/`Cell` model as the Excel workbooks, so `ExcelParser` works on it unchanged. +[`CsvReader`](xref:ExcelReader.Core.Reader.Csv.CsvReader) is a workbook with exactly one, unnamed sheet (`csv.FirstSheet`, the same sheet as `csv.Sheets[0]`). It streams RFC 4180 CSV (quoted fields, embedded delimiters/newlines, `""`-escaped quotes) through the same [`Row`](xref:ExcelReader.Core.Reader.Row)/[`Cell`](xref:ExcelReader.Core.Reader.Cell) model as the Excel workbooks, so [`ExcelParser`](xref:ExcelReader.Core.Parser.ExcelParser%601) works on it unchanged. + ```csharp using ExcelReader.Core.Parser; using ExcelReader.Core.Reader; @@ -25,19 +26,21 @@ foreach (var item in ExcelParser.FromAttributes().Parse(csv.FirstShee } ``` -`Excel.FromCsv`/`FromCsvFile`/`FromCsvAsync`/`FromCsvFileAsync` mirror the other formats' factory shape. Pass `CsvReaderOptions` to change the delimiter/quote character, supply a non-UTF-8 `Encoding` (transcoded to UTF-8 internally), or turn off BOM detection: +[`Excel.FromCsv`](xref:ExcelReader.Core.Reader.Excel.FromCsv*)/`FromCsvFile`/`FromCsvAsync`/`FromCsvFileAsync` mirror the other formats' factory shape and take the same [`ExcelReaderOptions`](xref:ExcelReader.Core.Reader.ExcelReaderOptions). Its `Csv` property, a [`CsvReaderOptions`](xref:ExcelReader.Core.Reader.Csv.CsvReaderOptions), changes the delimiter/quote character, supplies a non-UTF-8 `Encoding` (transcoded to UTF-8 internally), or turns off BOM detection; `MaxCellBytes` and `InternStrings` stay on `ExcelReaderOptions`, as for every format: + ```csharp -var options = new CsvReaderOptions { Delimiter = (byte)';' }; +var options = new ExcelReaderOptions { Csv = new CsvReaderOptions { Delimiter = (byte)';' } }; using var csv = Excel.FromCsvFile("relatorio.csv", options); ``` -Every CSV cell is text (`CellType.ExcelString`, or `CellType.Empty` for a blank field); at the reader level there is no binary numeric or date representation, so `Cell.TryGetDateTime`/`IsDate1904` (always `false` for CSV) do not apply. The typed parser, however, is CSV-specialized: `ExcelParser.Parse(CsvSheet)` parses `DateTime`/`DateOnly` columns directly from the cell text (ISO or culture format, honoring `Culture` — e.g. pt-BR `02/07/2026`), so no `[ExcelConverter]` is needed for dates. All the usual attributes work unchanged (`[ExcelColumn]` aliases, `[ExcelRequired]`, `[ExcelConverter]`), and a converter still takes precedence over the built-in date parsing. (Holding the sheet as an `IExcelSheet` instead routes through the generic Excel pipeline, where dates use serial-number semantics — prefer the concrete `Parse(CsvSheet)` overload for CSV.) +Every CSV cell is text ([`CellType.ExcelString`](xref:ExcelReader.Core.Reader.CellType.ExcelString), or [`CellType.Empty`](xref:ExcelReader.Core.Reader.CellType.Empty) for a blank field); at the reader level there is no binary numeric or date representation, so [`Cell.TryGetDateTime`](xref:ExcelReader.Core.Reader.Cell.TryGetDateTime*)/`IsDate1904` (always `false` for CSV) do not apply. The typed parser, however, is CSV-specialized: `ExcelParser.Parse(CsvSheet)` parses `DateTime`/`DateOnly` columns directly from the cell text (ISO or culture format, honoring `Culture` — e.g. pt-BR `02/07/2026`), so no `[ExcelConverter]` is needed for dates. All the usual attributes work unchanged (`[ExcelColumn]` aliases, `[ExcelRequired]`, `[ExcelConverter]`), and a converter still takes precedence over the built-in date parsing. (Holding the sheet as an [`IExcelSheet`](xref:ExcelReader.Core.Reader.IExcelSheet) instead routes through the generic Excel pipeline, where dates use serial-number semantics — prefer the concrete `Parse(CsvSheet)` overload for CSV.) A `CsvReader` over memory, a file path or a seekable stream allows several enumerators at once, each reading the sheet from the start. Over a seekable stream, every enumerator starts at the position the stream had when the `CsvReader` was created; where the stream's position ends up afterwards is unspecified, so do not rely on it (a file stream, and a memory stream with an exposed buffer, are read without touching it). A non-seekable stream can be read once: a second enumeration throws `InvalidOperationException`. -`Excel.Open`/`OpenAsync` do **not** auto-detect CSV — plain text has no magic-byte signature to sniff. Name the format instead: every `Open`/`OpenAsync` overload takes an optional `ExcelFileFormat`, and `ExcelReaderOptions.Csv` carries the dialect, so a caller that decides the format from an extension or a content-type header configures both families through one options object: +[`Excel.Open`](xref:ExcelReader.Core.Reader.Excel.Open*)/`OpenAsync` do **not** auto-detect CSV — plain text has no magic-byte signature to sniff. Name the format instead: every `Open`/`OpenAsync` overload takes an optional [`ExcelFileFormat`](xref:ExcelReader.Core.Reader.ExcelFileFormat), and [`ExcelReaderOptions.Csv`](xref:ExcelReader.Core.Reader.ExcelReaderOptions.Csv) carries the dialect, so a caller that decides the format from an extension or a content-type header configures both families through one options object: + ```csharp ExcelFileFormat format = Path.GetExtension(path) is ".csv" ? ExcelFileFormat.Csv : ExcelFileFormat.Unknown; @@ -48,34 +51,37 @@ using IExcelWorkbook workbook = Excel.Open(path, format, new ExcelReaderOptions }); ``` -`ExcelFileFormat.Unknown` keeps the signature-based detection the no-format overloads do; `EncryptedOoxml` is a detection result rather than something to open, so passing it throws `ArgumentOutOfRangeException`. +[`ExcelFileFormat.Unknown`](xref:ExcelReader.Core.Reader.ExcelFileFormat.Unknown) keeps the signature-based detection the no-format overloads do; `EncryptedOoxml` is a detection result rather than something to open, so passing it throws `ArgumentOutOfRangeException`. ## Sniff a CSV dialect -`CsvSniffer.Detect` infers the delimiter, quote character, and encoding (from a leading byte-order mark) from a sample of bytes, so a `;`-separated pt-BR export or a TSV can be read without the caller knowing the dialect up front. It is deterministic (ties break by candidate order) and never throws on arbitrary input — an indecisive sample returns `CsvDialect.Default` (comma, `"`, UTF-8). +[`CsvSniffer.Detect`](xref:ExcelReader.Core.Reader.Csv.CsvSniffer.Detect*) infers the delimiter, quote character, and encoding (from a leading byte-order mark) from a sample of bytes, so a `;`-separated pt-BR export or a TSV can be read without the caller knowing the dialect up front. It is deterministic (ties break by candidate order) and never throws on arbitrary input — an indecisive sample returns [`CsvDialect.Default`](xref:ExcelReader.Core.Reader.Csv.CsvDialect.Default) (comma, `"`, UTF-8). + ```csharp using ExcelReader.Core.Reader; using ExcelReader.Core.Reader.Csv; CsvDialect dialect = CsvSniffer.DetectFile("export.csv"); -using var csv = Excel.FromCsvFile("export.csv", CsvReaderOptions.Default.WithDialect(dialect)); +using var csv = Excel.FromCsvFile("export.csv", new ExcelReaderOptions { Csv = CsvReaderOptions.Default.WithDialect(dialect) }); ``` -`CsvSniffer.Detect` takes a span of bytes or a seekable `Stream`, `DetectFile` takes a path, and `DetectAsync`/`DetectFileAsync` are the async siblings; only the first 64 KiB are examined. The `Stream` overloads require a seekable source — they read a bounded sample and restore the stream's position — so a non-seekable stream throws `ArgumentException`; buffer it first, or pass the bytes as a span instead. Pass `CsvSnifferOptions` to change the candidate delimiters/quotes (and their priority order) or the number of sample lines considered. +`CsvSniffer.Detect` takes a span of bytes or a seekable `Stream`, `DetectFile` takes a path, and `DetectAsync`/`DetectFileAsync` are the async siblings; only the first 64 KiB are examined. The `Stream` overloads require a seekable source — they read a bounded sample and restore the stream's position — so a non-seekable stream throws `ArgumentException`; buffer it first, or pass the bytes as a span instead. Pass [`CsvSnifferOptions`](xref:ExcelReader.Core.Reader.Csv.CsvSnifferOptions) to change the candidate delimiters/quotes (and their priority order) or the number of sample lines considered. -`CsvReaderOptions.SniffDialect` does the sample-then-apply above for you, at the entry point that owns the source — the `Excel.FromCsv*` factories, `Excel.Open` with `ExcelFileFormat.Csv`, and the parallel CSV entry points: +[`CsvReaderOptions.SniffDialect`](xref:ExcelReader.Core.Reader.Csv.CsvReaderOptions.SniffDialect) does the sample-then-apply above for you, at the entry point that owns the source — the `Excel.FromCsv*` factories, and `Excel.Open` with [`ExcelFileFormat.Csv`](xref:ExcelReader.Core.Reader.ExcelFileFormat.Csv). The dialect of a parallel read is the one the CSV workbook was opened with: + ```csharp -using var csv = Excel.FromCsvFile("export.csv", CsvReaderOptions.Default with { SniffDialect = true }); +using var csv = Excel.FromCsvFile("export.csv", new ExcelReaderOptions { Csv = CsvReaderOptions.Default with { SniffDialect = true } }); ``` An explicitly set `Encoding` survives sniffing unless the source carries a byte-order mark naming a different one — a sniffed encoding only ever comes from a BOM, so it never silently overrides a caller who knows their file is Windows-1252. ## Write CSV -`CsvWorkbookWriter` emits RFC 4180 CSV: no styles or shared strings, so rows stream straight to the output. A CSV file is a single sheet, so `AddSheet` is called once (the name is ignored). +[`CsvWorkbookWriter`](xref:ExcelReader.Core.Writer.Csv.CsvWorkbookWriter) emits RFC 4180 CSV: no styles or shared strings, so rows stream straight to the output. A CSV file is a single sheet, so `AddSheet` is called once (the name is ignored). + ```csharp using ExcelReader.Core.Writer; using ExcelReader.Core.Writer.Csv; @@ -101,8 +107,9 @@ using (CsvRowWriter row = writer.StartRow()) Fields are quoted only when they contain the delimiter, quote character, `\r`, or `\n`; embedded quotes are doubled. `bool` writes as lowercase `true`/`false` and `DateTime`/`DateOnly` as round-trip ISO 8601 (`"O"`); `TimeOnly` as a time-of-day fraction — all matching what `ExcelParser.Parse(CsvSheet)` expects, so a file written by `CsvWorkbookWriter` parses back without configuration. `Skip(count)` writes empty fields to keep column positions aligned (CSV has no sparse-cell concept). -`CsvWriterOptions` mirrors `CsvReaderOptions` property for property, so a file written with one set of settings reads back with the matching set: +[`CsvWriterOptions`](xref:ExcelReader.Core.Writer.Csv.CsvWriterOptions) mirrors `CsvReaderOptions`'s delimiter, quote and encoding, so a file written with one set of settings reads back with the matching set: + ```csharp using var workbook = CsvWorkbookWriter.Create(stream, leaveOpen: false, new CsvWriterOptions { diff --git a/docs/guide/encryption.md b/docs/guide/encryption.md index d44b17b2..ad0cf358 100644 --- a/docs/guide/encryption.md +++ b/docs/guide/encryption.md @@ -7,15 +7,17 @@ Reading and writing password-protected packages. Password-protected `.xlsx`/`.xlsb`/`.xlsm` files open through the same entry points — the password goes on the options, so every overload supports it: + ```csharp var options = new ExcelReaderOptions { Password = "hunter2" }; using IExcelWorkbook workbook = Excel.Open("protected.xlsx", options); foreach (Row row in workbook.FirstSheet) { /* ... */ } ``` -`ExcelEncryptionException.Reason` tells you what to do about a failure — only `PasswordRequired` and +[`ExcelEncryptionException.Reason`](xref:ExcelReader.Core.Reader.ExcelEncryptionException.Reason) tells you what to do about a failure — only `PasswordRequired` and `PasswordIncorrect` are worth re-prompting for: + ```csharp try { @@ -37,13 +39,14 @@ different mechanism entirely, stored as hashes in the plaintext XML. only the buffers it owns: the derivation buffer and the derived key. **The `dataIntegrity` HMAC is not verified by default when reading from a stream.** -`ExcelReaderOptions.VerifyEncryptedIntegrity` defaults to `false`, because the HMAC covers the whole +[`ExcelReaderOptions.VerifyEncryptedIntegrity`](xref:ExcelReader.Core.Reader.ExcelReaderOptions.VerifyEncryptedIntegrity) defaults to `false`, because the HMAC covers the whole encrypted package: checking it means a full pass over the file before the first row, which makes time-to-first-row proportional to file size and defeats streaming. With it off, decryption still fails loudly on corrupt ciphertext (the plaintext stops being a valid ZIP), but *targeted* tampering by someone who can modify the file is not detected. Set it to `true` whenever the workbook comes from somewhere you do not control and you care that it was not altered: + ```csharp var options = new ExcelReaderOptions { @@ -61,9 +64,10 @@ that, reading is ~10% slower than reading the same workbook unencrypted — see [Encrypted workbooks](../performance/benchmarks.md#encrypted-workbooks) in the benchmarks. Writing an encrypted workbook is a second step: build the package with any writer, then wrap it with -`Excel.EncryptPackage`, which produces an agile-encrypted (ECMA-376 4.4) CFB container — AES-256-CBC, +[`Excel.EncryptPackage`](xref:ExcelReader.Core.Reader.Excel.EncryptPackage*), which produces an agile-encrypted (ECMA-376 4.4) CFB container — AES-256-CBC, SHA-512, 100,000 spin iterations, with a `dataIntegrity` HMAC. + ```csharp using ExcelReader.Core.Reader; using ExcelReader.Core.Writer; diff --git a/docs/guide/errors.md b/docs/guide/errors.md new file mode 100644 index 00000000..ff6fd667 --- /dev/null +++ b/docs/guide/errors.md @@ -0,0 +1,47 @@ +# Errors across languages + +The .NET library throws exceptions. The C ABI turns each failure into an `XL_*` status code, and the +Python, Rust and C++ bindings turn that code into their own error type. This page maps one onto the +other, so a failure seen in one language can be recognized in another. + +## Status codes + +Every `xl_*` function returns one of these. After a failure, `xl_last_error_ptr` returns the message +for the calling thread. + +| Code | Constant | Meaning | +|---|---|---| +| 0 | `XL_OK` | Success. | +| -1 | `XL_EOF` | No more rows or batches. Not an error: the bindings end iteration on it. | +| -2 | `XL_BUFFER_TOO_SMALL` | The caller's buffer cannot hold the result. The bindings grow the buffer and retry, so it never reaches user code. | +| -3 | `XL_INVALID_HANDLE` | The handle is null, already closed, or was never opened. | +| -4 | `XL_INVALID_ARGUMENT` | An argument failed validation before any work started: a null pointer, a bad length, malformed options. | +| -5 | `XL_ERROR` | Anything else. The message is the message of the .NET exception behind it. | +| -6 | `XL_STATUS_PASSWORD_REQUIRED` | The workbook is encrypted and no password was supplied. | +| -7 | `XL_STATUS_PASSWORD_INCORRECT` | The supplied password did not match the workbook's verifier. | + +## By failure + +| Failure | C# | Status | Python | Rust | C++ | +|---|---|---|---|---|---| +| Encrypted, no password | [`ExcelEncryptionException`](xref:ExcelReader.Core.Reader.ExcelEncryptionException), `Reason == PasswordRequired` | -6 | `PasswordRequiredError` | `Error::PasswordRequired` | `xl::Error`, `code == XL_STATUS_PASSWORD_REQUIRED` | +| Wrong password | `ExcelEncryptionException`, `Reason == PasswordIncorrect` | -7 | `PasswordIncorrectError` | `Error::PasswordIncorrect` | `xl::Error`, `code == XL_STATUS_PASSWORD_INCORRECT` | +| Unsupported encryption, or failed integrity check | `ExcelEncryptionException`, `Reason` is `UnsupportedScheme` or `IntegrityFailure` | -5 | `ExcelReaderError` | `Error::Native` | `xl::Error`, `code == XL_ERROR` | +| Corrupt or malformed file | `InvalidDataException` | -5 | `ExcelReaderError` | `Error::Native` | `xl::Error`, `code == XL_ERROR` | +| Feature the reader does not support | `NotSupportedException` | -5 | `ExcelReaderError` | `Error::Native` | `xl::Error`, `code == XL_ERROR` | +| File exceeds a configured resource limit | [`ExcelLimitExceededException`](xref:ExcelReader.Core.Reader.ExcelLimitExceededException) | -5 | `ExcelReaderError` | `Error::Native` | `xl::Error`, `code == XL_ERROR` | +| Workbook already closed | `ObjectDisposedException` | -3 | `ExcelReaderError` | `Error::Native` | `xl::Error`, `code == XL_INVALID_HANDLE` | +| Invalid argument | `ArgumentException` and its subclasses | -4, or -5 when the check happens past the ABI boundary | `ExcelReaderError` | `Error::Native` | `xl::Error` | + +`PasswordRequiredError` and `PasswordIncorrectError` derive from `ExcelReaderError`, so +`except ExcelReaderError` catches every native failure in Python. Rust's `Error::code()` returns the +status for every variant. The C++ header throws nothing: every entry point returns +`std::expected`. + +Only the two password cases are worth retrying with different input. The rest fail the same way on +every attempt. + +Each binding also checks some arguments itself, before calling the ABI, and reports those with its +own native types: Python raises `ValueError`, `TypeError` or `IndexError`; Rust panics on an +out-of-range index (see the crate README's "Bounds and panics"); C++ returns `xl::Error` with +`code == XL_INVALID_ARGUMENT`. diff --git a/docs/guide/migrating-to-v6.md b/docs/guide/migrating-to-v6.md index ca58d48e..3c368bb5 100644 --- a/docs/guide/migrating-to-v6.md +++ b/docs/guide/migrating-to-v6.md @@ -11,12 +11,12 @@ A v5 reader owned one "current sheet" that you moved with `MoveToSheet`; in v6 t | v5 | v6 | |---|---| -| `XlsxReader`, `XlsbReader`, `XlsReader` | `XlsxWorkbook`, `XlsbWorkbook`, `XlsWorkbook` | -| `XlsxReader.Enumerator` (and the other two) | `XlsxWorkbook.Enumerator` (and `XlsbWorkbook.Enumerator`, `XlsWorkbook.Enumerator`) | -| `IExcelRowReader` | `IExcelWorkbook` and `IExcelSheet` | -| `IExcelRowReader` | `IExcelSheet` | -| `ExcelSheet`, `reader.Sheets()` | `XlsxSheet` / `XlsbSheet` / `XlsSheet` / `CsvSheet`, `workbook.Sheets` | -| `CsvReader` | `CsvReader` (same name; now a workbook with one sheet) | +| `XlsxReader`, `XlsbReader`, `XlsReader` | [`XlsxWorkbook`](xref:ExcelReader.Core.Reader.Xlsx.XlsxWorkbook), [`XlsbWorkbook`](xref:ExcelReader.Core.Reader.Xlsb.XlsbWorkbook), [`XlsWorkbook`](xref:ExcelReader.Core.Reader.Xls.XlsWorkbook) | +| `XlsxReader.Enumerator` (and the other two) | [`XlsxWorkbook.Enumerator`](xref:ExcelReader.Core.Reader.Xlsx.XlsxWorkbook.Enumerator) (and [`XlsbWorkbook.Enumerator`](xref:ExcelReader.Core.Reader.Xlsb.XlsbWorkbook.Enumerator), [`XlsWorkbook.Enumerator`](xref:ExcelReader.Core.Reader.Xls.XlsWorkbook.Enumerator)) | +| `IExcelRowReader` | [`IExcelWorkbook`](xref:ExcelReader.Core.Reader.IExcelWorkbook) and [`IExcelSheet`](xref:ExcelReader.Core.Reader.IExcelSheet) | +| `IExcelRowReader` | [`IExcelSheet`](xref:ExcelReader.Core.Reader.IExcelSheet%601) | +| `ExcelSheet`, `reader.Sheets()` | [`XlsxSheet`](xref:ExcelReader.Core.Reader.Xlsx.XlsxSheet) / [`XlsbSheet`](xref:ExcelReader.Core.Reader.Xlsb.XlsbSheet) / [`XlsSheet`](xref:ExcelReader.Core.Reader.Xls.XlsSheet) / [`CsvSheet`](xref:ExcelReader.Core.Reader.Csv.CsvSheet), `workbook.Sheets` | +| [`CsvReader`](xref:ExcelReader.Core.Reader.Csv.CsvReader) | `CsvReader` (same name; now a workbook with one sheet) | An explicit `ExcelEnumerable` declaration becomes `ExcelEnumerable` (likewise for XLSX and XLSB). @@ -88,7 +88,7 @@ for (int i = 0; i < workbook.SheetCount; i++) } ``` -When you open a format directly (`Excel.FromXlsxFile` and the like), `workbook.Sheets` is a typed list: `foreach (XlsxSheet sheet in workbook.Sheets)` walks every sheet without boxing. +When you open a format directly ([`Excel.FromXlsxFile`](xref:ExcelReader.Core.Reader.Excel.FromXlsxFile*) and the like), `workbook.Sheets` is a typed list: `foreach (XlsxSheet sheet in workbook.Sheets)` walks every sheet without boxing. v5's `ExcelSheet` had a positional `Deconstruct`, so `foreach (var (i, name) in reader.Sheets())` worked. The v6 sheets do not deconstruct; read the properties instead: diff --git a/docs/guide/migrating-to-v7.md b/docs/guide/migrating-to-v7.md new file mode 100644 index 00000000..1a73bf9d --- /dev/null +++ b/docs/guide/migrating-to-v7.md @@ -0,0 +1,144 @@ +# Migrating from v6 to v7 + +What changed in parallel reading and in the rest of the public API, and how to port v6 code. +See also [reading.md](reading.md#parallel-reading-opt-in) for the full guide. + +## What changed + +Parallel reading now takes a sheet, and it works on XLSX and XLSB as well as CSV. `CsvParallel` is gone: its eighteen path, memory and stream methods became six [`SheetParallel`](xref:ExcelReader.Core.Reader.SheetParallel) methods that take any [`IExcelSheet`](xref:ExcelReader.Core.Reader.IExcelSheet). You open the workbook yourself, pass its sheet, and the library parallelises as far as the format allows. The contract types lost their `Csv` prefix because they now serve every format. + +## Types + +| v6 | v7 | +|---|---| +| `CsvParallel` | `SheetParallel` | +| `CsvParallelOptions` | [`SheetParallelOptions`](xref:ExcelReader.Core.Reader.SheetParallelOptions) (no `Reader` property) | +| `ICsvRecord` | [`IRowRecord`](xref:ExcelReader.Core.Reader.IRowRecord%601) | +| `ICsvAccumulator` | [`IRowAccumulator`](xref:ExcelReader.Core.Reader.IRowAccumulator%602) | +| `CsvAggregation` | [`RowAggregation`](xref:ExcelReader.Core.Reader.RowAggregation%601) | +| `CsvRowAction` | [`RowAction`](xref:ExcelReader.Core.Reader.RowAction%601) | + +The old names are removed, with no `[Obsolete]` forwarders. Members, contracts and results are unchanged. + +## Before and after + +A CSV file: + +```csharp +// v6 +await foreach (Sale s in CsvParallel.ParseAsync(path, parser, new CsvParallelOptions { DegreeOfParallelism = 8, Reader = options })) +{ + Total += s.Revenue; +} + +// v7 +using CsvReader csv = Excel.FromCsvFile(path, new ExcelReaderOptions { Csv = options }); +await foreach (Sale s in SheetParallel.ParseAsync(csv.FirstSheet, parser, new SheetParallelOptions { DegreeOfParallelism = 8 })) +{ + Total += s.Revenue; +} +``` + +CSV bytes in memory: + +```csharp +// v6 +await foreach (Sale s in CsvParallel.ParseAsync(data, parser, new CsvParallelOptions { DegreeOfParallelism = 8, Reader = options })) +{ + Total += s.Revenue; +} + +// v7 +using CsvReader csv = Excel.FromCsv(data, new ExcelReaderOptions { Csv = options }); +await foreach (Sale s in SheetParallel.ParseAsync(csv.FirstSheet, parser, new SheetParallelOptions { DegreeOfParallelism = 8 })) +{ + Total += s.Revenue; +} +``` + +A stream: + +```csharp +// v6 +await foreach (Sale s in CsvParallel.ParseAsync(stream, parser, new CsvParallelOptions { DegreeOfParallelism = 8, Reader = options })) +{ + Total += s.Revenue; +} + +// v7 +using CsvReader csv = Excel.FromCsv(stream, leaveOpen: true, new ExcelReaderOptions { Csv = options }); +await foreach (Sale s in SheetParallel.ParseAsync(csv.FirstSheet, parser, new SheetParallelOptions { DegreeOfParallelism = 8 })) +{ + Total += s.Revenue; +} +``` + +The other five methods port the same way: open a workbook, pass `csv.FirstSheet` (or any sheet) in place of the path, bytes or stream, and drop `Reader` from the options. XLSX and XLSB need no other change: + +```csharp +using XlsxWorkbook workbook = Excel.FromXlsxFile("big.xlsx"); +await foreach (Sale s in SheetParallel.ParseAsync(workbook.FirstSheet, parser, new SheetParallelOptions { DegreeOfParallelism = 8 })) +{ + Total += s.Revenue; +} +``` + +## Behavior changes + +- The caller owns the workbook and keeps it open until the task completes or the enumeration ends. v6 opened and closed its own source. +- The CSV dialect is given when the workbook is opened ([`ExcelReaderOptions.Csv`](xref:ExcelReader.Core.Reader.ExcelReaderOptions.Csv)), not in the parallel options. +- A CSV in a `MemoryStream` is no longer partitioned: the stream overload of [`Excel.FromCsv`](xref:ExcelReader.Core.Reader.Excel.FromCsv*) reads it sequentially. Pass the bytes as `ReadOnlyMemory` to keep partitioning. +- `HeaderRow` rules are the same as v6. `ParseAsync` takes the header from [`ExcelParserOptions.HeaderRow`](xref:ExcelReader.Core.Parser.ExcelParserOptions.HeaderRow) and rejects a non-zero [`SheetParallelOptions.HeaderRow`](xref:ExcelReader.Core.Reader.SheetParallelOptions.HeaderRow); the parser-based `ForEachAsync` and `AggregateAsync` use `SheetParallelOptions.HeaderRow`. +- XLSX and XLSB sheets are now read on several threads when `DegreeOfParallelism` allows it. XLS, a table view (`ExcelTable.AsSheet()`), a sheet under 256 KB uncompressed and a CSV that cannot be partitioned still read sequentially with the same results. +- `PrefetchDecompression` does not apply to a parallel XLSX or XLSB read; the pipeline already overlaps inflate and parse. + +## Other breaking changes + +These changes replace positional parameters and loose arguments with types that can grow without another break. + +| v6 | v7 | +|---|---| +| `ExcelParserConfig` | [`ExcelParserOptions`](xref:ExcelReader.Core.Parser.ExcelParserOptions); the [`ExcelParser`](xref:ExcelReader.Core.Parser.ExcelParser) factories' parameter is `options` | +| `IExcelCellConverter.TryConvert(in Cell, bool isDate1904, IFormatProvider provider, out T)` | `TryConvert(in Cell, in ExcelRowContext context, out T)` | +| `ExcelCellReader`, `ExcelRowParser`, `ExcelCellReaders.*`: `(…, bool isDate1904, IFormatProvider provider, …)` | `(…, in ExcelRowContext context, …)` | +| `Excel.InferSchema(sheet, headerRow, sampleSize, parseText)` returning `ExcelColumnSchema[]` | `Excel.InferSchema(sheet, new SchemaInferenceOptions { HeaderRow = …, SampleSize = …, ParseText = … })` returning `IReadOnlyList` | +| `new ExcelDataReader(sheet, headerRow)` | `new SheetDataReader(sheet, new SheetDataReaderOptions { HeaderRow = … })` | +| `sheet.ToArrowRecordBatch(schema, headerRow)` | `sheet.ToArrowRecordBatch(new ArrowReadOptions { Schema = schema, HeaderRow = … })` | +| `WriteRecordBatch(batch, sheetName, writeHeader)` and the async form | `WriteRecordBatch(batch, new ArrowWriteOptions { SheetName = …, WriteHeader = … })` | +| `XlsWorkbookWriter.Create(stream, leaveOpen, date1904)` | `XlsWorkbookWriter.Create(stream, leaveOpen, new XlsWriterOptions { Date1904 = … })` | +| `Excel.FromCsv*(…, CsvReaderOptions options)` | `Excel.FromCsv*(…, new ExcelReaderOptions { Csv = options })` | +| `CsvReaderOptions.MaxCellBytes`, `CsvReaderOptions.InternStrings` | [`ExcelReaderOptions.MaxCellBytes`](xref:ExcelReader.Core.Reader.ExcelReaderOptions.MaxCellBytes), [`ExcelReaderOptions.InternStrings`](xref:ExcelReader.Core.Reader.ExcelReaderOptions.InternStrings) | +| `CellType.Formula` | Removed. A formula cell reports its cached result's type: `Number`, `Date`, `ExcelString`, `Boolean` or `Error` | +| `ExcelDataReader` (its name collided with Sylvan.Data.Excel and the ExcelDataReader package) | [`SheetDataReader`](xref:ExcelReader.Core.Reader.SheetDataReader) | +| `Cell.TryGetDateTime(out DateTime)`, which silently assumed the 1900 date system | `Cell.TryGetDateTime(workbook.IsDate1904, out DateTime)` | + +A converter reads the date system and culture from the context: + +```csharp +// v6 +public bool TryConvert(in Cell cell, bool isDate1904, IFormatProvider provider, out decimal value) + => decimal.TryParse(cell.GetString(), NumberStyles.Currency, provider, out value); + +// v7 +public bool TryConvert(in Cell cell, in ExcelRowContext context, out decimal value) + => decimal.TryParse(cell.GetString(), NumberStyles.Currency, context.FormatProvider, out value); +``` + +To unit-test a converter, build the context with an initializer: `new ExcelRowContext { IsDate1904 = false, FormatProvider = CultureInfo.InvariantCulture }`. + +Two behavior changes come with them: + +- [`Excel.Open`](xref:ExcelReader.Core.Reader.Excel.Open*) with [`ExcelFileFormat.Csv`](xref:ExcelReader.Core.Reader.ExcelFileFormat.Csv) now applies `ExcelReaderOptions.MaxCellBytes` and `InternStrings`. v6 ignored both and used the CSV defaults. +- Formula cells are typed by their cached result in every format. v6 reported an XLSX text result and an XLS numeric result as `Formula`, so an XLS formula with a date style lost its `Date` type, and XLSB never used `Formula`. +- The CLI `convert` command no longer shifts dates by four years when the source workbook uses the 1904 date system. +- [`XlsxWriterOptions`](xref:ExcelReader.Core.Writer.Xlsx.XlsxWriterOptions) gains `Date1904`, which v6 lacked. It defaults to `false`, so existing output is unchanged. + +## Native bindings + +The C ABI is unchanged: no signature moved, and `XL_ABI_VERSION` stays 6. Behavior changed. `degree_of_parallelism` of `0` on `xl_parse_typed` and the Arrow exports now parallelises XLSX and XLSB as well as CSV, using every processor. Pass `1` for the old sequential behavior. The Rust, Python and C++ bindings take the same value (`degree_of_parallelism`, `parallelism`) and need no code change; a `parallelism=0` call on an XLSX or XLSB sheet is now faster, and holds every partition's columns until they merge, so its peak memory is higher. + +`XL_CELL_FORMULA` (5) is no longer produced; formula cells come back with their result's cell type. The constant stays defined in the header and in the bindings. + +`xl_write_options.date1904` now applies to XLSX as well as XLSB and XLS; v6 ignored it for XLSX. + +`xl_csv_aggregate` keeps its signature. Inside, it opens a [`CsvReader`](xref:ExcelReader.Core.Reader.Csv.CsvReader) with the dialect from the native struct and calls `SheetParallel`. diff --git a/docs/guide/parsing.md b/docs/guide/parsing.md index 0f4214f3..badfac10 100644 --- a/docs/guide/parsing.md +++ b/docs/guide/parsing.md @@ -7,6 +7,7 @@ generator, the fluent API, converters, and the zero-copy `ref struct` path. `ExcelParser.FromAttributes()` maps worksheet columns to the public settable properties of `T`. Columns match on the property name, or on `[ExcelColumn("header")]` aliases — repeat the attribute to accept several headers. The first row is the header by default. + ```csharp using ExcelReader.Core.Parser; using ExcelReader.Core.Reader; @@ -29,10 +30,11 @@ foreach (var item in parser.Parse(workbook.FirstSheet)) } ``` -Built-in property types: `string`, `bool`, `DateTime`, `DateOnly`, `TimeOnly`, `TimeSpan`, `DateTimeOffset`, `Guid`, `char`, every integral and floating type plus `decimal`, and `enum`s (matched by member name or numeric value). Each also works as a `Nullable`. Empty cells leave the property at its default; an unparseable cell is skipped (keeps the default) unless the column is required. `T` needs no parameterless-constructor constraint, so models with `required` members are supported. +Built-in property types: `string`, `bool`, `DateTime`, `DateOnly`, `TimeOnly`, `TimeSpan`, `DateTimeOffset`, `Guid`, `char`, every integral and floating type plus `decimal`, and `enum`s (matched by member name or numeric value). Each also works as a `Nullable`. Empty cells leave the property at its default; an unparseable cell is skipped (keeps the default) unless the column is required. `T` needs no parameterless-constructor constraint, so models with `required` members are supported, through both `FromAttributes` and the source generator. `required` alone does not make a column mandatory; that is what `[ExcelRequired]` does. -`Parse` also accepts the `IExcelSheet` of the `IExcelWorkbook` from `Excel.Open`, so you can parse without knowing the concrete format: +`Parse` also accepts the [`IExcelSheet`](xref:ExcelReader.Core.Reader.IExcelSheet) of the [`IExcelWorkbook`](xref:ExcelReader.Core.Reader.IExcelWorkbook) from [`Excel.Open`](xref:ExcelReader.Core.Reader.Excel.Open*), so you can parse without knowing the concrete format: + ```csharp using IExcelWorkbook workbook = Excel.Open("changes.xlsx"); // or .xlsb / .xls foreach (var item in ExcelParser.FromAttributes().Parse(workbook.FirstSheet)) { /* ... */ } @@ -40,8 +42,9 @@ foreach (var item in ExcelParser.FromAttributes().Parse(workbook.Firs ## Generate typed maps at compile time (Native AOT / trimming) -`ExcelParser.FromAttributes` and `ExcelRecordLayout.FromAttributes` reflect over `T` (`GetProperties`, `MakeGenericMethod`), which trimming can break and Native AOT cannot run at all. The raw `Excel.From*` readers already use no reflection; mark a model `[ExcelSerializable]` to get the same guarantee for the typed layer — a source generator emits a compile-time map from the model's own `[ExcelColumn]`/`[ExcelRequired]`/`[ExcelConverter]`/`[ExcelIgnore]` attributes, and `ExcelParser.Generated`/`ExcelRecordLayout.Generated` read and write through that map instead of reflection: +[`ExcelParser.FromAttributes`](xref:ExcelReader.Core.Parser.ExcelParser.FromAttributes*) and [`ExcelRecordLayout.FromAttributes`](xref:ExcelReader.Core.Writer.ExcelRecordLayout.FromAttributes*) reflect over `T` (`GetProperties`, `MakeGenericMethod`), which trimming can break and Native AOT cannot run at all. The raw `Excel.From*` readers already use no reflection; mark a model `[ExcelSerializable]` to get the same guarantee for the typed layer — a source generator emits a compile-time map from the model's own `[ExcelColumn]`/`[ExcelRequired]`/`[ExcelConverter]`/`[ExcelIgnore]` attributes, and [`ExcelParser.Generated`](xref:ExcelReader.Core.Parser.ExcelParser.Generated*)/[`ExcelRecordLayout.Generated`](xref:ExcelReader.Core.Writer.ExcelRecordLayout.Generated*) read and write through that map instead of reflection: + ```csharp using ExcelReader.Core.Parser; using ExcelReader.Core.Reader; @@ -66,8 +69,8 @@ foreach (var item in ExcelParser.Generated().Parse(workbook.FirstShee var changes = new[] { new ChangeRow { File = "README.md", LinesAdded = 12 } }; await using var stream = File.Create("changes.xlsx"); -await using var workbook = XlsxWorkbookWriter.Create(stream); // or XlsbWorkbookWriter / XlsWorkbookWriter / CsvWorkbookWriter -await using (var sheet = workbook.AddSheet("Changes")) +await using var writer = XlsxWorkbookWriter.Create(stream); // or XlsbWorkbookWriter / XlsWorkbookWriter / CsvWorkbookWriter +await using (var sheet = writer.AddSheet("Changes")) { await sheet.WriteRecordsAsync(changes, ExcelRecordLayout.Generated()); } @@ -76,15 +79,16 @@ await using (var sheet = workbook.AddSheet("Changes")) Notes: - `[ExcelSerializable]` requires the model — and every type it's nested inside, if any — to be `partial`; the generator emits into an additional part of the same declaration. A compile error (`EXR001`/`EXR002`) names exactly what to fix. -- Supported property types match `ExcelParser.FromAttributes`'s: `string`, `bool`, `DateTime`, `DateOnly`, `TimeOnly`, `TimeSpan`, `DateTimeOffset`, `Guid`, `char`, every integral and floating type plus `decimal`, `enum`s, and `Nullable` of each — plus `ReadOnlySpan`, which binds zero-copy through `ExcelCellReaders.Utf8`. The model may be a `ref struct`: mark it `ref partial struct` and the generator emits `IExcelRowMap` on it like any other shape, so `ExcelParser.Generated` and `ExcelParser.Build` read it AOT-clean, as do the map-driven `CsvParallel.ForEachAsync`/`CsvParallel.AggregateAsync` overloads. +- Supported property types match `ExcelParser.FromAttributes`'s: `string`, `bool`, `DateTime`, `DateOnly`, `TimeOnly`, `TimeSpan`, `DateTimeOffset`, `Guid`, `char`, every integral and floating type plus `decimal`, `enum`s, and `Nullable` of each — plus `ReadOnlySpan`, which binds zero-copy through [`ExcelCellReaders.Utf8`](xref:ExcelReader.Core.Parser.ExcelCellReaders.Utf8*). The model may be a `ref struct`: mark it `ref partial struct` and the generator emits [`IExcelRowMap`](xref:ExcelReader.Core.Parser.IExcelRowMap%601) on it like any other shape, so `ExcelParser.Generated` and [`ExcelParser.Build`](xref:ExcelReader.Core.Parser.ExcelParser.Build*) read it AOT-clean, as do the map-driven [`SheetParallel.ForEachAsync`](xref:ExcelReader.Core.Reader.SheetParallel.ForEachAsync*)/[`SheetParallel.AggregateAsync`](xref:ExcelReader.Core.Reader.SheetParallel.AggregateAsync*) overloads. - The generator requires a build via `dotnet build`/the .NET SDK. Visual Studio's or `MSBuild.exe`'s .NET Framework host can't load it, so a project built only through those tools won't see generated code — build via the SDK, or fall back to `ExcelParser.FromAttributes`/`ExcelRecordLayout.FromAttributes` for that build path. -- `ExcelParser.Generated` builds one map per model and reuses it for every reader, including CSV — unlike `ExcelParser.FromAttributes`, which swaps in a text-based date reader specifically for CSV. A `[ExcelSerializable]` model reads `DateTime`/`DateOnly`/`TimeOnly` via `ExcelCellReaders.DateTimeAuto`/`DateOnlyAuto`/`TimeOnlyAuto`: an Excel serial number first, falling back to date/time text when the cell isn't numeric — so a CSV column round-trips through the library's own writer either way. The one edge case this can't distinguish: a CSV cell that's only digits (e.g. an Excel serial number typed as plain text) is always read as a serial number, never as date text. +- `ExcelParser.Generated` builds one map per model and reuses it for every reader, including CSV — unlike `ExcelParser.FromAttributes`, which swaps in a text-based date reader specifically for CSV. A `[ExcelSerializable]` model reads `DateTime`/`DateOnly`/`TimeOnly` via [`ExcelCellReaders.DateTimeAuto`](xref:ExcelReader.Core.Parser.ExcelCellReaders.DateTimeAuto)/`DateOnlyAuto`/`TimeOnlyAuto`: an Excel serial number first, falling back to date/time text when the cell isn't numeric — so a CSV column round-trips through the library's own writer either way. The one edge case this can't distinguish: a CSV cell that's only digits (e.g. an Excel serial number typed as plain text) is always read as a serial number, never as date text. - The attribute-based reflection path keeps working unchanged; `[ExcelSerializable]` is an additive, opt-in alternative for the same model shape, not a replacement. ## Map columns at runtime (fluent API) -`[ExcelColumn]`/`[ExcelRequired]` fix the mapping at compile time. When the mapping itself is a runtime decision — loaded from a config file, chosen by a user in a UI, or different per input file — build it with `ExcelRowMapBuilder` through `ExcelParser.Build` instead: +`[ExcelColumn]`/`[ExcelRequired]` fix the mapping at compile time. When the mapping itself is a runtime decision — loaded from a config file, chosen by a user in a UI, or different per input file — build it with [`ExcelRowMapBuilder`](xref:ExcelReader.Core.Parser.ExcelRowMapBuilder%601) through `ExcelParser.Build` instead: + ```csharp using ExcelReader.Core.Parser; using ExcelReader.Core.Reader; @@ -111,6 +115,7 @@ The map is built once, when `Build` is called, from a fresh builder instance — Bind a fixed column index instead of a header name with `PropertyAt` — for files with no header row at all: + ```csharp var parser = ExcelParser.Build(builder => builder .Factory(() => new ChangeRow()) @@ -120,8 +125,9 @@ var parser = ExcelParser.Build(builder => builder A builder that uses `PropertyAt` skips the header-row step entirely — the first row is already data. Mixing `PropertyAt` with `Property`/`PropertyNullable`/`Converted` on the same builder throws, since one map can't both wait for a header row and skip it. -`ExcelParser.BuildWithAttributeFallback` merges the builder's bindings with `[ExcelColumn]`/`[ExcelRequired]`-driven ones reflected from `T`: a builder binding replaces every attribute-driven property that shares one of its header names; a property whose header names none of the builder's bindings mention keeps its attribute-driven behavior. Useful for overriding just the one column that's a runtime decision without redeclaring the whole model — reuse one of the property's existing `[ExcelColumn]` names in the builder: +[`ExcelParser.BuildWithAttributeFallback`](xref:ExcelReader.Core.Parser.ExcelParser.BuildWithAttributeFallback*) merges the builder's bindings with `[ExcelColumn]`/`[ExcelRequired]`-driven ones reflected from `T`: a builder binding replaces every attribute-driven property that shares one of its header names; a property whose header names none of the builder's bindings mention keeps its attribute-driven behavior. Useful for overriding just the one column that's a runtime decision without redeclaring the whole model — reuse one of the property's existing `[ExcelColumn]` names in the builder: + ```csharp var parser = ExcelParser.BuildWithAttributeFallback(builder => builder .Property(["file"], ExcelCellReaders.String, (ref ChangeRow r, string v) => r.File = v.ToUpperInvariant())); @@ -133,13 +139,14 @@ The match is by header name, not by property identity: configuring a *different* ## Parser configuration -Pass an `ExcelParserConfig` to control header handling and culture: +Pass an [`ExcelParserOptions`](xref:ExcelReader.Core.Parser.ExcelParserOptions) to control header handling and culture: + ```csharp using System.Globalization; using ExcelReader.Core.Parser; -var config = new ExcelParserConfig +var options = new ExcelParserOptions { HeaderRow = 1, // 1-based row holding the headers ColumnNameComparer = StringComparer.OrdinalIgnoreCase, @@ -147,15 +154,16 @@ var config = new ExcelParserConfig Culture = CultureInfo.GetCultureInfo("pt-BR"), // parse "1.234,56" as 1234.56m }; -var parser = ExcelParser.FromAttributes(config); +var parser = ExcelParser.FromAttributes(options); ``` -`Culture` applies when parsing text-backed numeric/`Guid` cells (XLSX inline and shared strings); binary numeric cells (XLS/XLSB) carry a raw value and ignore it. `HeaderNormalization` flags (`Trim`, `CollapseSpaces`, `RemoveDiacritics`) are applied to both the sheet headers and the property names before matching. +`Culture` applies when parsing text-backed numeric/`Guid` cells (XLSX inline and shared strings); binary numeric cells (XLS/XLSB) carry a raw value and ignore it. [`HeaderNormalization`](xref:ExcelReader.Core.Parser.HeaderNormalization) flags (`Trim`, `CollapseSpaces`, `RemoveDiacritics`) are applied to both the sheet headers and the property names before matching. ## Required columns Mark a property `[ExcelRequired]` to assert its column exists and carries a value: + ```csharp public sealed class Order { @@ -173,8 +181,9 @@ public sealed class Order ## Custom converters -For types the built-in parsers do not handle — money strings, custom formats, domain value objects — implement `IExcelCellConverter` and attach it with `[ExcelConverter]`. `T` must be the property's exact type. One instance is created and reused across all rows, so converters must be stateless. +For types the built-in parsers do not handle — money strings, custom formats, domain value objects — implement [`IExcelCellConverter`](xref:ExcelReader.Core.Parser.IExcelCellConverter%601) and attach it with `[ExcelConverter]`. `T` must be the property's exact type. One instance is created and reused across all rows, so converters must be stateless. + ```csharp using System.Globalization; using ExcelReader.Core.Parser; @@ -182,7 +191,7 @@ using ExcelReader.Core.Reader; public sealed class BrlMoneyConverter : IExcelCellConverter { - public bool TryConvert(in Cell cell, bool isDate1904, IFormatProvider provider, out decimal value) + public bool TryConvert(in Cell cell, in ExcelRowContext context, out decimal value) { string text = cell.GetString().Replace("R$", "", StringComparison.Ordinal).Trim(); return decimal.TryParse(text, NumberStyles.Currency, CultureInfo.GetCultureInfo("pt-BR"), out value); @@ -196,12 +205,13 @@ public sealed class Invoice } ``` -Return `false` to signal a parse failure (the property keeps its default). Empty cells are skipped before the converter runs. +Return `false` to signal a parse failure (the property keeps its default). Empty cells are skipped before the converter runs. `context` carries the workbook's date system (`IsDate1904`) and the parser culture (`FormatProvider`); to unit-test a converter, pass `new ExcelRowContext { IsDate1904 = false, FormatProvider = CultureInfo.InvariantCulture }`. ## Parse into a ref struct (zero-copy) `ExcelParser.FromAttributes` takes a `ref struct` model just as it takes a class or a struct — same attribute-driven column matching (`[ExcelColumn]`, `[ExcelRequired]`, `[ExcelConverter]`), but a `ReadOnlySpan` property binds directly to the cell's raw bytes instead of allocating a `string`: + ```csharp using System.Text; using ExcelReader.Core.Parser; @@ -223,6 +233,7 @@ foreach (ChangeRowRef item in ExcelParser.FromAttributes().Parse(w The result of `Parse` supports `await foreach` (add `.WithCancellation(ct)` to pass a token), so a `ref struct` model can be parsed asynchronously — the rows are streamed via `MoveNextAsync` while the model stays a zero-copy `ref struct`: + ```csharp await using var workbook = await Excel.FromXlsxFileAsync("changes.xlsx"); @@ -236,6 +247,6 @@ A few differences from a class or struct model: - **Span fields alias the reader's row buffer** — valid only until the next row. Copy them out (e.g. `Encoding.UTF8.GetString(span)`) if you need to keep the value past the loop body. Under `await foreach`, the same rule means the model can't be held across an `await` in the loop body. - **The non-generic `IEnumerator.Current` throws.** `IEnumerable` and `IAsyncEnumerable` both accept a `ref struct` element type on net9.0+, so the sequence flows through them normally; only the legacy non-generic `IEnumerator.Current` cannot, because it would have to box. LINQ over a `ref struct` model is still out, since its operators do not carry the `allows ref struct` anti-constraint. -- **Not AOT/trim-safe** on this path, since it reflects over `T`'s properties and compiles setters at runtime. For an AOT-clean zero-copy model, mark the `ref partial struct` `[ExcelSerializable]` and read it with `ExcelParser.Generated` — see [Generate the map at compile time](#generate-the-map-at-compile-time-aot-clean). +- **Not AOT/trim-safe** on this path, since it reflects over `T`'s properties and compiles setters at runtime. For an AOT-clean zero-copy model, mark the `ref partial struct` `[ExcelSerializable]` and read it with `ExcelParser.Generated` — see [Generate typed maps at compile time](#generate-typed-maps-at-compile-time-native-aot--trimming). - A regular `struct`/`class` model uses the same entry point — only a genuine `ref struct` model gets the extra zero-copy span-property binding. diff --git a/docs/guide/reading.md b/docs/guide/reading.md index ad838027..dcacb011 100644 --- a/docs/guide/reading.md +++ b/docs/guide/reading.md @@ -5,6 +5,7 @@ See also [parsing.md](parsing.md) for binding rows to types, and [csv.md](csv.md ## Read rows + ```csharp using ExcelReader.Core.Reader; @@ -26,59 +27,107 @@ foreach (var row in workbook.FirstSheet) } ``` -An opened workbook (`XlsxWorkbook`, `XlsbWorkbook`, `XlsWorkbook`, `CsvReader`) hands out its sheets: `FirstSheet` is the first one, `Sheets[i]` any of them. A sheet holds no resources of its own; iterating it starts a new, independent read. Dispose the workbook, not the sheets. +An opened workbook ([`XlsxWorkbook`](xref:ExcelReader.Core.Reader.Xlsx.XlsxWorkbook), [`XlsbWorkbook`](xref:ExcelReader.Core.Reader.Xlsb.XlsbWorkbook), [`XlsWorkbook`](xref:ExcelReader.Core.Reader.Xls.XlsWorkbook), [`CsvReader`](xref:ExcelReader.Core.Reader.Csv.CsvReader)) hands out its sheets: `FirstSheet` is the first one, `Sheets[i]` any of them. A sheet holds no resources of its own; iterating it starts a new, independent read. Dispose the workbook, not the sheets. -### Parallel CSV parsing (opt-in) +### Parallel reading (opt-in) +[`SheetParallel`](xref:ExcelReader.Core.Reader.SheetParallel) reads one sheet on several threads. It takes any sheet, so it works the same on XLSX, XLSB and CSV: + + ```csharp -await foreach (var row in CsvParallel.ParseAsync("big.csv", ExcelParser.FromAttributes(), new CsvParallelOptions { DegreeOfParallelism = 8 })) +using XlsxWorkbook workbook = Excel.FromXlsxFile("big.xlsx"); + +await foreach (var row in SheetParallel.ParseAsync(workbook.FirstSheet, ExcelParser.FromAttributes(), new SheetParallelOptions { DegreeOfParallelism = 8 })) { Total += row.Revenue; } ``` -Rows arrive in file order, identical to the sequential parser. Parsing and type conversion run across -threads; whatever your loop body does per row does not — if that dominates, raising the degree will -not help. Sources that cannot be partitioned (non-seekable streams, non-UTF-8 encodings, small files) -fall back to sequential parsing with the same results. +You own the workbook: keep it open until the task completes or the enumeration ends. For CSV, give the dialect when you open it (`Excel.FromCsvFile(path, options)`), not in the options. `DegreeOfParallelism` of `0` (the default) uses every processor; `1` reads sequentially. + +There are three families, each over a sheet: + +| Family | Result | Order | +|---|---|---| +| `ParseAsync` | an `IAsyncEnumerable` | the sheet's own order, identical to `parser.Parse(sheet)` | +| `ForEachAsync` | calls your callback on the worker threads | none | +| `AggregateAsync` | folds one accumulator per partition into a result | the same result as a single-threaded run | + +Parsing and type conversion run across threads; whatever your loop body does per row does not, so if that dominates, raising the degree will not help. -For a fold or a per-row side effect, `CsvParallel.ForEachAsync` accepts a `ref struct` row type, so text -columns stay as `ReadOnlySpan` and the per-row model allocation disappears entirely — a class -model still costs one instance per row: +Both of the next two examples read a `ref struct` model, so text columns stay as `ReadOnlySpan` and the per-row model allocation disappears; a class model still costs one instance per row. +A fold over an XLSX sheet, with one accumulator per partition merged in sheet order: + + ```csharp public ref struct SaleRow { [ExcelColumn("Region")] public ReadOnlySpan Region { get; set; } - [ExcelColumn("Units")] public int Units { get; set; } + [ExcelColumn("Units")] public int Units { get; set; } +} + +public sealed class UnitTotal : IRowAccumulator +{ + public long Units; + + public void Add(SaleRow model) => Units += model.Units; + + public void Merge(UnitTotal following) => Units += following.Units; } +using XlsxWorkbook workbook = Excel.FromXlsxFile("big.xlsx"); + +UnitTotal total = await SheetParallel.AggregateAsync( + workbook.FirstSheet, + ExcelParser.FromAttributes(), + new SheetParallelOptions { HeaderRow = 1 }); +``` + +The same call takes a [`RowAggregation`](xref:ExcelReader.Core.Reader.RowAggregation%601) when you would rather pass delegates than write a type, and an `IRowRecord` type in place of the parser. + +A per-row callback: + + +```csharp +using XlsxWorkbook workbook = Excel.FromXlsxFile("big.xlsx"); + long total = 0; -await CsvParallel.ForEachAsync( - "big.csv", +await SheetParallel.ForEachAsync( + workbook.FirstSheet, ExcelParser.FromAttributes(), row => Interlocked.Add(ref total, row.Units), - new CsvParallelOptions { DegreeOfParallelism = 8, HeaderRow = 1 }); + new SheetParallelOptions { DegreeOfParallelism = 8, HeaderRow = 1 }); ``` -Unlike `CsvParallel.ParseAsync`, the callback runs on the worker threads, so records arrive in no -particular order and the callback must be safe to call from several threads at once. Spans are valid -only for the duration of the call. - -Delivery is at least once, not exactly once: the callback may be invoked more than once for the same -record. A partition whose start offset was guessed wrongly is read again from the confirmed offset, and -the discarded pass may already have delivered a whole partition's worth of records — 1 MiB to 64 MiB of them, not just a few near the seam. -On a source with quoted fields those records may also be misparsed, carrying values that appear nowhere -in the file, and an exception the callback threw during a discarded pass is discarded with it. A start -is only guessed wrongly inside a quoted field, so a source in which the quote character never appears -delivers every record exactly once; for that guarantee on any source, use `CsvParallel.AggregateAsync`, -where the discarded partition's accumulator is thrown away, and apply side effects once it returns. -A callback that writes to a database or sends messages must be idempotent and must also tolerate -records that do not exist — deduplicating on a key does not filter those out. - -See the [parallel CSV benchmarks](../performance/benchmarks.md#parallel-csv) for what this buys. Those figures are -`CsvParallel.AggregateAsync`'s, measured over the same projection path this shares: at dop 16 it is -~2.8x faster than the typed path while allocating ~410x less, with zero garbage collections. +Unlike `ParseAsync`, the callback runs on the worker threads, so records arrive in no particular order and the callback must be safe to call from several threads at once. Spans are valid only for the duration of the call. + +`HeaderRow` follows the same rule as before: `ParseAsync` takes the header from the parser's [`ExcelParserOptions.HeaderRow`](xref:ExcelReader.Core.Parser.ExcelParserOptions.HeaderRow) and rejects a non-zero [`SheetParallelOptions.HeaderRow`](xref:ExcelReader.Core.Reader.SheetParallelOptions.HeaderRow), while the parser-based `ForEachAsync` and `AggregateAsync` use `SheetParallelOptions.HeaderRow`. + +#### When it reads sequentially + +These cases read on one thread with the same results and no error: + +- `DegreeOfParallelism` of 1. +- A sheet under 256 KB uncompressed. +- XLS. +- A table view (`ExcelTable.AsSheet()`). +- A CSV that cannot be partitioned: a stream that is not a file, or a non-UTF-8 encoding. Only in-memory data (`ReadOnlyMemory`, `byte[]`) and a `FileStream`-backed source partition; a CSV over a `MemoryStream` does not, so pass its bytes as `ReadOnlyMemory`. + +`PrefetchDecompression` does not apply on this path: XLSX and XLSB go through a pipeline that already overlaps inflate and parse. + +#### Delivery guarantees + +- **`ParseAsync` and `AggregateAsync`** are exact on every format. `ParseAsync` yields the same items in the same order as the sequential parse; `AggregateAsync` equals its own single-threaded run. +- **XLSB** delivers every row exactly once, in `ForEachAsync` too. +- **XLSX** delivers exactly once for a well-formed sheet. The pipeline cuts the inflated sheet at what looks like a row start; on a sheet where those bytes sit inside cell content (a CDATA run, a comment) or the file is truncated, a worker detects it, and the read recovers by reading the rest sequentially. `ParseAsync` and `AggregateAsync` stay exact. In `ForEachAsync`, workers may already have called the callback for chunks that are then dropped, so on such a sheet rows can be delivered twice, or be rows the sequential reader would not produce. +- **CSV** is at least once in `ForEachAsync`, unchanged from v6: the callback may be invoked more than once for the same record. A partition whose start offset was guessed wrongly is read again from the confirmed offset, and the discarded pass may already have delivered a whole partition's worth of records, 1 MiB to 64 MiB of them, not just a few near the seam. On a source with quoted fields those records may also be misparsed, carrying values that appear nowhere in the file, and an exception the callback threw during a discarded pass is discarded with it. A start is only guessed wrongly inside a quoted field, so a source in which the quote character never appears delivers every record exactly once. For that guarantee on any source, use `AggregateAsync`, where the discarded partition's accumulator is thrown away, and apply side effects once it returns. + +A callback that writes to a database or sends messages must be idempotent on CSV, and on XLSX if the sheet may be malformed, and must also tolerate records that do not exist: deduplicating on a key does not filter those out. + +[`ExcelParseException.Row`](xref:ExcelReader.Core.Parser.ExcelParseException.Row) on `ParseAsync` counts the rows emitted so far. On `ForEachAsync` and `AggregateAsync` it is the exact row when the read is sequential and `0` when it was partitioned. + +See the [parallel sheet benchmarks](../performance/benchmarks.md#parallel-sheet-reading) and the [parallel CSV benchmarks](../performance/benchmarks.md#parallel-csv) for what this buys. On a real-data XLSX the sheet stage drops from 29 ms (prefetched, sequential) to about 18 ms, and on XLSB from 16 ms to about 13 ms; the floor is inflate, which runs on one thread. CSV has no such floor: its `AggregateAsync` at dop 16 is ~2.8x faster than the typed path while allocating ~410x less, with zero garbage collections. Properties need setters. A get-only property is skipped during binding and silently receives nothing. @@ -88,6 +137,7 @@ Properties need setters. A get-only property is skipped during binding and silen dotnet add package ExcelReader.Arrow ``` + ```csharp using ExcelReader.Arrow; using ExcelReader.Core.Reader; @@ -96,10 +146,11 @@ using var workbook = Excel.FromXlsxFile("report.xlsx"); Apache.Arrow.RecordBatch batch = workbook.FirstSheet.ToArrowRecordBatch(); ``` -`schema` defaults to `Excel.InferSchema`'s guess; pass an explicit `ExcelColumnSchema[]` to skip inference. The whole sheet is materialized into one `RecordBatch` — there is no chunked/streaming variant yet. Inference costs about the same time and ~1.8x the allocation of an explicit schema on the same file. See [Arrow conversion](../performance/benchmarks.md#arrow-conversion) in the benchmarks. +The schema defaults to [`Excel.InferSchema`](xref:ExcelReader.Core.Reader.Excel.InferSchema*)'s guess; pass one in `new ArrowReadOptions { Schema = ... }` to skip inference, and set `HeaderRow` there (default 1, 0 for none). The whole sheet is materialized into one `RecordBatch` — there is no chunked/streaming variant yet. Inference costs about the same time and ~1.8x the allocation of an explicit schema on the same file. See [Arrow conversion](../performance/benchmarks.md#arrow-conversion) in the benchmarks. `WriteRecordBatch`/`WriteRecordBatchAsync` are the write-side mirror — one call from a `RecordBatch` to a sheet, for XLSX, XLSB, XLS, and CSV: + ```csharp using ExcelReader.Arrow; using ExcelReader.Core.Writer; @@ -109,12 +160,13 @@ using XlsxWorkbookWriter workbook = XlsxWorkbookWriter.Create(File.Create("repor workbook.WriteRecordBatch(batch); // adds the sheet, writes the header + every row, ends the workbook ``` -Supports string, large string, string view, every integer width (signed and unsigned), half float, float, double, decimal128, decimal256, boolean, date32, date64, time32, time64, timestamp and null columns; any other type (binary, nested, dictionary) throws `NotSupportedException`. Pass `writeHeader: false` to skip the header row, or a `sheetName` (XLSX/XLSB/XLS only — CSV has no sheet name) to rename it from the `"Sheet1"` default. +Supports string, large string, string view, every integer width (signed and unsigned), half float, float, double, decimal128, decimal256, boolean, date32, date64, time32, time64, timestamp and null columns; any other type (binary, nested, dictionary) throws `NotSupportedException`. Pass `new ArrowWriteOptions { WriteHeader = false }` to skip the header row, or set `SheetName` (XLSX/XLSB/XLS only — CSV has no sheet name) to rename it from the `"Sheet1"` default. ## Open by auto-detecting the format -`Excel.Open` picks the workbook type from the file signature (XLSX/XLSB are ZIP packages, XLS is an OLE2 document) and returns an `IExcelWorkbook`. `FirstSheet` gives the first sheet, and a sheet iterates its rows directly, so no pattern-match is needed for basic row iteration. +[`Excel.Open`](xref:ExcelReader.Core.Reader.Excel.Open*) picks the workbook type from the file signature (XLSX/XLSB are ZIP packages, XLS is an OLE2 document) and returns an [`IExcelWorkbook`](xref:ExcelReader.Core.Reader.IExcelWorkbook). `FirstSheet` gives the first sheet, and a sheet iterates its rows directly, so no pattern-match is needed for basic row iteration. + ```csharp using ExcelReader.Core.Reader; @@ -126,8 +178,9 @@ foreach (var row in workbook.FirstSheet) } ``` -`SheetCount`, `SheetAt(index)` and `TryGetSheet(name)` are on `IExcelWorkbook` itself, so you can walk every sheet without knowing the format. Each `IExcelSheet` carries its own `Index`, `Name` and `Visibility`: +`SheetCount`, `SheetAt(index)` and `TryGetSheet(name)` are on `IExcelWorkbook` itself, so you can walk every sheet without knowing the format. Each [`IExcelSheet`](xref:ExcelReader.Core.Reader.IExcelSheet) carries its own `Index`, `Name` and `Visibility`: + ```csharp using IExcelWorkbook workbook = Excel.Open("report.xlsx"); @@ -144,6 +197,7 @@ for (int i = 0; i < workbook.SheetCount; i++) `TryGetSheet` matches a name ignoring case and opens nothing: + ```csharp if (workbook.TryGetSheet("Totals", out var totals)) { @@ -154,10 +208,11 @@ if (workbook.TryGetSheet("Totals", out var totals)) } ``` -When you open a format directly (`Excel.FromXlsxFile` and the like) the workbook's `Sheets` is a typed, indexable list (`Sheets.Count`, `Sheets[i]`, `foreach`) of `XlsxSheet`, `XlsbSheet` or `XlsSheet` values, so walking it boxes nothing. Reaching a sheet only looks up a name and an index; the file is not read until you iterate the sheet. +When you open a format directly ([`Excel.FromXlsxFile`](xref:ExcelReader.Core.Reader.Excel.FromXlsxFile*) and the like) the workbook's `Sheets` is a typed, indexable list (`Sheets.Count`, `Sheets[i]`, `foreach`) of [`XlsxSheet`](xref:ExcelReader.Core.Reader.Xlsx.XlsxSheet), [`XlsbSheet`](xref:ExcelReader.Core.Reader.Xlsb.XlsbSheet) or [`XlsSheet`](xref:ExcelReader.Core.Reader.Xls.XlsSheet) values, so walking it boxes nothing. Reaching a sheet only looks up a name and an index; the file is not read until you iterate the sheet. `Visibility` reports whether a sheet is shown in the workbook's tab bar, read from the format's own encoding of it (XLSX's `state` attribute, XLSB's `BrtBundleSh.hsState`, XLS's `BoundSheet8.hsState`). It is reported, never enforced — a hidden sheet enumerates its rows like any other, so converters and exporters filter on it themselves: + ```csharp using ExcelReader.Core; // ExcelSheetVisibility, shared by the readers and writers @@ -172,22 +227,55 @@ for (int i = 0; i < workbook.SheetCount; i++) } ``` -`ExcelSheetVisibility.VeryHidden` is the state Excel's own unhide dialog does not offer; both it and `Hidden` come back here. A sheet whose format says nothing about its state — or says something no producer agrees on — reads as `Visible`. +[`ExcelSheetVisibility.VeryHidden`](xref:ExcelReader.Core.ExcelSheetVisibility.VeryHidden) is the state Excel's own unhide dialog does not offer; both it and `Hidden` come back here. A sheet whose format says nothing about its state — or says something no producer agrees on — reads as `Visible`. CSV is exposed as a single, unnamed sheet (`SheetCount == 1`, `Name == ""`, always `Visible`). Pattern-match to the concrete type only for workbook-specific internals beyond this surface. `OpenAsync` is the async counterpart. Both require a seekable stream (or a file path) so the signature can be read without consuming the input. -Detection covers the signed formats only. To open a source whose format you already know — including CSV, which has no signature to detect — pass an `ExcelFileFormat` and let `ExcelReaderOptions.Csv` carry the dialect; see [CSV](csv.md#read-csv). +Detection covers the signed formats only. To open a source whose format you already know — including CSV, which has no signature to detect — pass an [`ExcelFileFormat`](xref:ExcelReader.Core.Reader.ExcelFileFormat) and let [`ExcelReaderOptions.Csv`](xref:ExcelReader.Core.Reader.ExcelReaderOptions.Csv) carry the dialect; see [CSV](csv.md#read-csv). + ```csharp using IExcelWorkbook workbook = Excel.Open("report.csv", ExcelFileFormat.Csv); ``` +## Read an Excel table + +A workbook's tables (ranges formatted with **Format as Table**) are listed on `Tables`, ordered by sheet +and creation, and found by name with `TryGetTable`, ignoring case. XLSX/XLSM and XLSB report them; XLS +and CSV always report none. `AsSheet()` reads only the table: its header row first, then its data rows +without the totals row, each cut to the table's columns, so it plugs into anything that takes a sheet. + + +```csharp +using IExcelWorkbook workbook = Excel.Open("sales.xlsx"); +if (workbook.TryGetTable("Sales", out ExcelTable? sales)) +{ + foreach (ProductRow row in ExcelParser.FromAttributes().Parse(sales.AsSheet())) + { + Console.WriteLine($"{row.Product}: {row.Qty}"); + } +} +``` + +- `FirstRow`, `FirstColumn`, `LastRow` and `LastColumn` are zero-based and include the header and totals + rows; `Ref` is the same range in A1 notation. +- Column indexes inside `AsSheet()` rows stay those of the sheet: a table starting at column C reports + its first cell at index 2. `Excel.InferSchema` keeps those indexes in [`ExcelColumnSchema.Index`](xref:ExcelReader.Core.Reader.Schema.ExcelColumnSchema.Index) but + lists only the table's columns. [`SheetDataReader`](xref:ExcelReader.Core.Reader.SheetDataReader) ordinals are relative to the table: ordinal 0 is + its first column. +- A table whose header row is turned off (`HeaderRowCount == 0`) still enumerates, but `Parse` throws + `InvalidOperationException` for every map, and `SheetDataReader` and `InferSchema` throw it when + `HeaderRow` is greater than 0. Read it with `new SheetDataReader(table.AsSheet(), new SheetDataReaderOptions { HeaderRow = 0 })`, + `Excel.InferSchema(table.AsSheet(), new SchemaInferenceOptions { HeaderRow = 0 })`, or by column index on the rows. +- `StyleName` is the table style's name, such as `TableStyleMedium2`, or `null` when the table has none. + ## Reading sheets in parallel -Each sheet of a workbook can be read on its own thread: +This is one thread per sheet. To read a single sheet on several threads, see [Parallel reading](#parallel-reading-opt-in). Each sheet of a workbook can be read on its own thread: + ```csharp using XlsxWorkbook workbook = Excel.FromXlsxFile(path); @@ -213,6 +301,7 @@ Disposing the workbook closes it to new sheets and enumerators; its file or stre Every sheet supports `await foreach`. For XLSX files, the async enumerator buffers one row at a time and uses the same row parser as the synchronous enumerator, so sync and async reads stay behaviorally aligned while awaits happen only when more bytes are needed. + ```csharp using ExcelReader.Core.Reader; @@ -224,10 +313,11 @@ await foreach (var row in workbook.FirstSheet) } ``` -`await foreach` binds to the sheet's `GetAsyncEnumerator(CancellationToken ct = default)` by pattern. The call itself does no I/O: opening the sheet part (and, for XLSX, loading shared strings) happens asynchronously on the first `MoveNextAsync`. Because `Row` and `Cell` are `ref struct` types, the current row cannot be held across an `await` inside the loop body: read its cells (or copy the values out) before awaiting anything else. +`await foreach` binds to the sheet's `GetAsyncEnumerator(CancellationToken ct = default)` by pattern. The call itself does no I/O: opening the sheet part (and, for XLSX, loading shared strings) happens asynchronously on the first `MoveNextAsync`. Because [`Row`](xref:ExcelReader.Core.Reader.Row) and [`Cell`](xref:ExcelReader.Core.Reader.Cell) are `ref struct` types, the current row cannot be held across an `await` inside the loop body: read its cells (or copy the values out) before awaiting anything else. `await foreach` does not accept `.WithCancellation(ct)`: `Row` being a `ref struct` rules out `IAsyncEnumerable`, so the loop binds to the pattern rather than the interface. To pass a token, or to `await` while a row is in scope, drive the enumerator manually: + ```csharp await using var workbook = await Excel.FromXlsxFileAsync("report.xlsx", ct: cancellationToken); await using var rows = workbook.FirstSheet.GetAsyncEnumerator(cancellationToken); @@ -244,11 +334,12 @@ Errors from opening the sheet, such as cancellation or a corrupt part, surface f ## Prefetch decompression (XLSX/XLSB) XLSX and XLSB are ZIP-backed, and inflating a sheet's compressed bytes competes for -wall-clock time with parsing it. `ExcelReaderOptions.PrefetchDecompression` overlaps the +wall-clock time with parsing it. [`ExcelReaderOptions.PrefetchDecompression`](xref:ExcelReader.Core.Reader.ExcelReaderOptions.PrefetchDecompression) overlaps the two: a background thread inflates ahead while the calling thread parses. It is **opt-in, defaults to `false`**, and only affects XLSX/XLSB — XLS and CSV have nothing to decompress, so the option is silently ignored for them. + ```csharp var options = new ExcelReaderOptions { PrefetchDecompression = true }; using var workbook = Excel.FromXlsxFile("report.xlsx", options); @@ -284,20 +375,21 @@ Writing has the same option, under a different name: see ## Bridge to ADO.NET (`IDataReader`) -`ExcelDataReader` adapts one `IExcelSheet` to `System.Data.IDataReader`, so it drops straight into `SqlBulkCopy`, `DataTable.Load`, Dapper, or any other ADO.NET consumer: +`SheetDataReader` adapts one `IExcelSheet` to `System.Data.IDataReader`, so it drops straight into `SqlBulkCopy`, `DataTable.Load`, Dapper, or any other ADO.NET consumer: + ```csharp using System.Data; using ExcelReader.Core.Reader; using IExcelWorkbook workbook = Excel.Open("report.xlsx"); -using IDataReader data = new ExcelDataReader(workbook.FirstSheet); // headerRow: 1 by default +using IDataReader data = new SheetDataReader(workbook.FirstSheet); // header on row 1 by default var table = new DataTable(); table.Load(data); ``` -The header row (1-based, default 1) fixes the column shape; pass `headerRow: 0` for a header-less sheet, whose columns come back named `Column0`, `Column1`, ... sized from the first data row. There is no schema-inference pass — `GetFieldType`/`GetValue`/the typed getters read the *current* row's own cell type, so a consumer building its schema from the first row (like `DataTable.Load`) locks in that row's types for the whole load. `NextResult()` always returns `false`; build one `ExcelDataReader` per sheet instead of chaining result sets. +The header row (1-based, default 1) fixes the column shape; pass `new SheetDataReaderOptions { HeaderRow = 0 }` for a header-less sheet, whose columns come back named `Column0`, `Column1`, ... sized from the first data row. There is no schema-inference pass — `GetFieldType`/`GetValue`/the typed getters read the *current* row's own cell type, so a consumer building its schema from the first row (like `DataTable.Load`) locks in that row's types for the whole load. `NextResult()` always returns `false`; build one `SheetDataReader` per sheet instead of chaining result sets. The bridge costs ~1.3–1.4x over iterating rows directly, and `GetBytes` stays within ~10% of the direct read. See [ADO.NET bridge](../performance/benchmarks.md#adonet-bridge-idatareader) in the benchmarks. diff --git a/docs/guide/untrusted-input.md b/docs/guide/untrusted-input.md new file mode 100644 index 00000000..ce61e4fa --- /dev/null +++ b/docs/guide/untrusted-input.md @@ -0,0 +1,139 @@ +# Reading untrusted files + +What one hostile workbook can cost you, which options bound it, and how to reject a file cleanly. +Read this before you accept spreadsheets from uploads, APIs or mail. + +A spreadsheet is a container whose sizes, counts and offsets are written by whoever made the file. +The readers check every one of those values before it drives an allocation, and the options below +turn what is left — a file that is well-formed but simply enormous — into an +[`ExcelLimitExceededException`](xref:ExcelReader.Core.Reader.ExcelLimitExceededException) instead of an out-of-memory crash. The defaults suit +a desktop tool or a batch job reading files you trust. A server should set its own. + +## The limits + +All of them live on [`ExcelReaderOptions`](xref:ExcelReader.Core.Reader.ExcelReaderOptions): + +| Option | Default | Bounds | Formats | +|---|---:|---|---| +| `MaxTotalDecompressedBytes` | 512 MiB | What one read of a sheet inflates or loads from its container, counting the workbook's metadata, styles, shared strings and decrypted package | XLSX, XLSB, XLS, encrypted | +| `MaxSharedStringBytes` | 128 MiB | The shared-string table, which is held in memory for the whole read | XLSX, XLSB, XLS | +| `MaxCellBytes` | 32 MiB | Each buffer that grows to hold one cell value, or one CSV record | all | +| `MaxZipEntries` | 65,536 | Entries in the ZIP directory | XLSX, XLSB | +| `MaxPasswordSpinCount` | 100,000 | Key-derivation iterations an agile-encrypted file may ask for | encrypted XLSX, XLSB | + +Two limits are fixed by Excel's grid rather than configured: a cell past column XFD (16,384) throws +with `LimitName == "Columns"`, and no single buffer may exceed .NET's array size +(`LimitName == "ArrayMaxLength"`). + +How they count: + +- **`MaxTotalDecompressedBytes` is per read of a sheet.** The workbook's one-time loads (its + metadata, styles and shared strings) count toward every read, and each read adds the sheet it + inflates. Reading a sheet again starts from the one-time loads, so a workbook kept open and read + repeatedly never runs out. +- **XLS has nothing to inflate, but it is not streamed:** the whole Workbook stream is read into + memory, up to `MaxTotalDecompressedBytes`. +- **CSV has no total limit.** It streams, so the size of what you hand it is the bound; only + `MaxCellBytes` applies. +- **Zero or a negative value turns a byte or entry limit off.** `MaxPasswordSpinCount` has no off + switch. In the Python, Rust and C++ bindings and the C ABI, `0` means the default instead, and a + limit cannot be turned off. +- **Reads running at once each have their own budget.** Ten uploads read together, or several sheets + of one workbook read on different threads, can inflate ten times the budget between them. + +## A profile for uploads + +Size each limit for the largest file you legitimately expect, measured rather than guessed. For a +service taking spreadsheets of a few MB, something like: + + +```csharp +var uploadOptions = new ExcelReaderOptions +{ + MaxTotalDecompressedBytes = 64L * 1024 * 1024, // everything one workbook may inflate + MaxSharedStringBytes = 16L * 1024 * 1024, + MaxCellBytes = 1024 * 1024, + MaxZipEntries = 1_000, +}; +``` + +Cap the upload itself as well, before it reaches the reader: the options bound what the file +expands to, not how many bytes you accept. [`Excel.Open`](xref:ExcelReader.Core.Reader.Excel.Open*) needs a seekable stream, so +buffer a request body to a file or to memory first. + +Leave [`PrefetchDecompression`](xref:ExcelReader.Core.Reader.ExcelReaderOptions.PrefetchDecompression) off: it adds a thread per read, which a busy server +cannot spare. For encrypted uploads, set `VerifyEncryptedIntegrity` so a tampered package is +rejected; see [Encrypted workbooks](encryption.md). + +No option bounds wall-clock time. The work a read does grows with the bytes it inflates, which the +limits bound; for a deadline, use the async entry points and enumerators, which take a +`CancellationToken`. + +## Rejecting a file + +A limit can trip when the workbook opens, when you start reading a sheet (XLSX loads its shared +strings then), or halfway down it. Wrap the whole read, not just the open, and do not commit rows until the read +has finished: + + +```csharp +try +{ + using IExcelWorkbook workbook = Excel.Open(upload, options: uploadOptions); + foreach (Row row in workbook.FirstSheet) + { + // stage the row; commit only once the loop has finished + } +} +catch (ExcelLimitExceededException ex) +{ + Console.WriteLine($"Rejected: {ex.LimitName} reached {ex.Actual}, limit {ex.Limit}"); +} +catch (Exception ex) when (IsRejectedFile(ex)) +{ + Console.WriteLine($"Rejected: {ex.Message}"); +} + +static bool IsRejectedFile(Exception ex) => + ex is InvalidDataException or EndOfStreamException or NotSupportedException + or ExcelEncryptionException or ExcelParseException + || ex.GetType() == typeof(ArgumentException) + || ex.GetType() == typeof(InvalidOperationException); +``` + +`LimitName` is the option's name (or `Columns`, `ArrayMaxLength`); `Limit` and `Actual` are in that +limit's unit — bytes, entries or iterations. + +That exception list is exactly what the fuzz suite accepts as a correct rejection of arbitrary +bytes. Anything else escaping a read of a file — an `IndexOutOfRangeException`, an +`OutOfMemoryException`, a hang — is a bug; please report it privately as described in +[SECURITY.md](../../SECURITY.md). See [Errors across languages](errors.md) for how these map onto the +bindings. + +## What the reader does not do + +- **It evaluates nothing.** A formula cell reports the result Excel cached when it saved the file, + so the value is whatever the file claims; recompute anything you rely on. Macros are never run + and external links never followed. +- **It resolves no XML entities beyond the predefined ones.** Sheet XML goes through the library's + own scanner, which decodes `&`-style and numeric references and copies anything else through + as text, so there is no entity expansion and no external fetch. The one `XmlReader`, for the + encryption descriptor, prohibits DTDs. + +## Writing untrusted text to CSV + +The writers store text as text: in XLSX, XLSB and XLS a string that starts with `=` is still a +string. CSV has no cell types, and Excel opening a `.csv` treats a field starting with `=`, `+`, +`-` or `@` as a formula. [`CsvWorkbookWriter`](xref:ExcelReader.Core.Writer.Csv.CsvWorkbookWriter) writes your text unchanged, so if a person may open the +file in a spreadsheet program, neutralize those fields yourself: + + +```csharp +static string Neutralize(string text) => + text.Length > 0 && "=+-@\t\r".Contains(text[0]) ? "'" + text : text; + +row.Write(Neutralize(comment)); +``` + +The leading `'` changes the data, so do this only for files meant for people, not for files another +program parses back. diff --git a/docs/guide/writing.md b/docs/guide/writing.md index e2e9e6c8..650c0236 100644 --- a/docs/guide/writing.md +++ b/docs/guide/writing.md @@ -5,6 +5,7 @@ typed records, and prefetch compression. See [csv.md](csv.md) for CSV output. ## Write XLSX workbooks + ```csharp using ExcelReader.Core.Writer; using ExcelReader.Core.Writer.Xlsx; @@ -35,28 +36,31 @@ await workbook.EndAsync(); By default, the XLSX writer emits inline strings to keep memory usage flat while rows stream out. If your workbook repeats many strings and smaller files matter more than the extra lookup table, opt in to shared strings: + ```csharp await using var workbook = XlsxWorkbookWriter.Create(stream, options: new XlsxWriterOptions { UseSharedStrings = true }); ``` ## Hidden sheets -Every `IWorkbookWriter` takes an optional `ExcelSheetVisibility` alongside the sheet name, written to whatever the format uses for it — XLSX's `state` attribute, XLSB's `BrtBundleSh.hsState`, XLS's `BoundSheet8.hsState` — so it reads back through [`IExcelSheet.Visibility`](reading.md#open-by-auto-detecting-the-format): +Every [`IWorkbookWriter`](xref:ExcelReader.Core.Writer.IWorkbookWriter%601) takes an optional [`ExcelSheetVisibility`](xref:ExcelReader.Core.ExcelSheetVisibility) alongside the sheet name, written to whatever the format uses for it — XLSX's `state` attribute, XLSB's `BrtBundleSh.hsState`, XLS's `BoundSheet8.hsState` — so it reads back through [`IExcelSheet.Visibility`](reading.md#open-by-auto-detecting-the-format): + ```csharp using ExcelReader.Core; // ExcelSheetVisibility, shared by the readers and writers -await using var data = workbook.AddSheet("Data"); // visible -await using var lookups = workbook.AddSheet("Lookups", ExcelSheetVisibility.Hidden); -await using var audit = workbook.AddSheet("Audit", ExcelSheetVisibility.VeryHidden); // not in Excel's unhide dialog +await using (var data = workbook.AddSheet("Data")) { /* rows */ } // visible +await using (var lookups = workbook.AddSheet("Lookups", ExcelSheetVisibility.Hidden)) { /* rows */ } +await using (var audit = workbook.AddSheet("Audit", ExcelSheetVisibility.VeryHidden)) { /* rows */ } // not in Excel's unhide dialog ``` A workbook whose sheets are *all* hidden is rejected when it is ended (`InvalidOperationException`): the formats allow it, but Excel reports such a file as damaged, so it is caught at write time rather than shipped to a user. CSV accepts the argument and ignores it — delimited text has no tab bar to hide from. ## Cell styles on write -Every `IWorkbookWriter` supports column- and row-level styling: a number format (currency, date, percentage), bold, and italic. Register a `CellStyle` once with `AddStyle` and apply its returned index to a column (before the first row) or to a whole row (when starting it): +Every `IWorkbookWriter` supports column- and row-level styling: a number format (currency, date, percentage), bold, and italic. Register a [`CellStyle`](xref:ExcelReader.Core.Writer.CellStyle) once with `AddStyle` and apply its returned index to a column (before the first row) or to a whole row (when starting it): + ```csharp using ExcelReader.Core.Writer; using ExcelReader.Core.Writer.Xlsx; @@ -90,8 +94,9 @@ Cell-level styling (one specific cell rather than a whole column or row) is out ## Read and write XLSB workbooks (BIFF12) -Use `Excel.FromXlsbFile`, `Excel.FromXlsb`, `Excel.FromXlsbFileAsync`, or `Excel.FromXlsbAsync` to open XLSB directly. For writing, use `XlsbWorkbookWriter`, `XlsbSheetWriter`, and `XlsbRowWriter`. +Use [`Excel.FromXlsbFile`](xref:ExcelReader.Core.Reader.Excel.FromXlsbFile*), [`Excel.FromXlsb`](xref:ExcelReader.Core.Reader.Excel.FromXlsb*), [`Excel.FromXlsbFileAsync`](xref:ExcelReader.Core.Reader.Excel.FromXlsbFileAsync*), or [`Excel.FromXlsbAsync`](xref:ExcelReader.Core.Reader.Excel.FromXlsbAsync*) to open XLSB directly. For writing, use [`XlsbWorkbookWriter`](xref:ExcelReader.Core.Writer.Xlsb.XlsbWorkbookWriter), [`XlsbSheetWriter`](xref:ExcelReader.Core.Writer.Xlsb.XlsbSheetWriter), and [`XlsbRowWriter`](xref:ExcelReader.Core.Writer.Xlsb.XlsbRowWriter). + ```csharp using ExcelReader.Core.Writer; using ExcelReader.Core.Writer.Xlsb; @@ -112,12 +117,13 @@ await using (XlsbSheetWriter sheet = workbook.AddSheet("Summary")) await workbook.EndAsync(); ``` -The XLSB writer also defaults to inline string cells. Set `XlsbWriterOptions.UseSharedStrings` to deduplicate repeated text into `sharedStrings.bin`. +The XLSB writer also defaults to inline string cells. Set [`XlsbWriterOptions.UseSharedStrings`](xref:ExcelReader.Core.Writer.Xlsb.XlsbWriterOptions.UseSharedStrings) to deduplicate repeated text into `sharedStrings.bin`. ## Write XLS workbooks (BIFF8) -`XlsWorkbookWriter` emits a binary BIFF8 `.xls` file. The sheet and row APIs are synchronous; only the final `EndAsync` (which assembles and flushes the OLE container) is async. BIFF8 is capped at 65,536 rows × 256 columns per sheet. +[`XlsWorkbookWriter`](xref:ExcelReader.Core.Writer.Xls.XlsWorkbookWriter) emits a binary BIFF8 `.xls` file. The sheet and row APIs are synchronous; only the final `EndAsync` (which assembles and flushes the OLE container) is async. BIFF8 is capped at 65,536 rows × 256 columns per sheet. + ```csharp using ExcelReader.Core.Writer; using ExcelReader.Core.Writer.Xls; @@ -147,8 +153,9 @@ await workbook.EndAsync(); ## Write typed records -The low-level writers above give you cell-by-cell control. When you just want to dump a collection of objects to a sheet, `sheet.WriteRecordsAsync(records, layout)` writes a header row followed by one row per record, with the columns coming from an `ExcelRecordLayout`. `ExcelRecordLayout.FromAttributes()` maps each public readable property to a column by reflection. It is an extension on `ISheetWriter`, so the same call works on a sheet from any of the writers above — XLSX, XLSB, XLS, and CSV. +The low-level writers above give you cell-by-cell control. When you just want to dump a collection of objects to a sheet, `sheet.WriteRecordsAsync(records, layout)` writes a header row followed by one row per record, with the columns coming from an [`ExcelRecordLayout`](xref:ExcelReader.Core.Writer.ExcelRecordLayout%601). `ExcelRecordLayout.FromAttributes()` maps each public readable property to a column by reflection. It is an extension on [`ISheetWriter`](xref:ExcelReader.Core.Writer.ISheetWriter%601), so the same call works on a sheet from any of the writers above — XLSX, XLSB, XLS, and CSV. + ```csharp using ExcelReader.Core.Writer; using ExcelReader.Core.Writer.Xlsx; @@ -175,18 +182,57 @@ await using (var sheet = workbook.AddSheet("Sales")) } ``` -Disposing the sheet ends it; add the next sheet after that (a duplicate name throws, ignoring case), so one workbook can hold sheets of different record types. The sheet stays open after `WriteRecordsAsync`, so you can append more rows before disposing it. `CsvWorkbookWriter` is the exception: a CSV file is a single sheet, so a second `AddSheet` throws (the sheet name is ignored). An `IAsyncEnumerable` overload streams records that are produced asynchronously. The written file round-trips straight back through `ExcelParser` because the headers are the property names. +Disposing the sheet ends it; add the next sheet after that (a duplicate name throws, ignoring case), so one workbook can hold sheets of different record types. The sheet stays open after `WriteRecordsAsync`, so you can append more rows before disposing it. [`CsvWorkbookWriter`](xref:ExcelReader.Core.Writer.Csv.CsvWorkbookWriter) is the exception: a CSV file is a single sheet, so a second `AddSheet` throws (the sheet name is ignored). An `IAsyncEnumerable` overload streams records that are produced asynchronously. The written file round-trips straight back through [`ExcelParser`](xref:ExcelReader.Core.Parser.ExcelParser%601) because the headers are the property names. Column behavior mirrors the parser attributes: - **`[ExcelColumn("Header")]`** — use a custom header instead of the property name (the first alias wins). - **`[ExcelIgnore]`** — exclude a property from both writing and parsing (for computed/transient members). -- **`[ExcelConverter(typeof(MyConverter))]`** — if the converter also implements `IExcelCellWriter`, it controls how the value is written, so a custom type round-trips through the same converter it reads with. +- **`[ExcelConverter(typeof(MyConverter))]`** — if the converter also implements [`IExcelCellWriter`](xref:ExcelReader.Core.Writer.IExcelCellWriter%601), it controls how the value is written, so a custom type round-trips through the same converter it reads with. `DateTime` and `DateOnly` are written as Excel date serials; `TimeOnly` as a time-of-day fraction. Numeric properties (including `Half`) become number cells; any other type is written as text formatted with the invariant culture — `DateTimeOffset` as ISO 8601 (`"O"`), enums by name — so the file parses back the same whatever the writing machine's culture. `Int128`/`UInt128` stay text because a number cell is a double and would drop their low digits. (`CsvWorkbookWriter` follows the CSV rules instead — see [Write CSV](csv.md#write-csv) — writing `DateTime`/`DateOnly` as ISO text and `TimeOnly` as a time-of-day fraction, all still round-tripping through `ExcelParser`.) For a model marked `[ExcelSerializable]`, pass `ExcelRecordLayout.Generated()` instead — same behavior, but driven by the source-generated map instead of reflection, so it stays Native AOT/trim-safe. See [Generate typed maps at compile time](parsing.md#generate-typed-maps-at-compile-time-native-aot--trimming). +## Write an Excel table + +The XLSX and XLSB sheet writers implement [`ITableSheetWriter`](xref:ExcelReader.Core.Writer.ITableSheetWriter%601). `BeginTable` writes the header row +and opens a table there; every row written until `EndTable` (or the end of the sheet) belongs to it. The +file opens in Excel already formatted as a table, with filter buttons and the chosen built-in style, and +reads back through [`IExcelWorkbook.Tables`](xref:ExcelReader.Core.Reader.IExcelWorkbook.Tables). + + +```csharp +using XlsxWorkbookWriter workbook = XlsxWorkbookWriter.Create(File.Create("report.xlsx")); +XlsxSheetWriter sheet = workbook.AddSheet("Report"); +sheet.BeginTable("Sales", ["Region", "Units", "Revenue"]); +foreach (Sale s in sales) +{ + using XlsxRowWriter row = sheet.StartRow(); + row.Write(s.Region); + row.Write(s.Units); + row.Write(s.Revenue); +} +sheet.EndTable(); +``` + +For records, `WriteTableAsync` does all three steps with the same layouts as `WriteRecordsAsync`: + + +```csharp +await sheet.WriteTableAsync(sales, "Sales", ExcelRecordLayout.FromAttributes()); +``` + +- [`ExcelTableOptions`](xref:ExcelReader.Core.Writer.ExcelTableOptions) sets `FirstColumn`, `StyleName` (a built-in `TableStyleLight1`–`21`, + `TableStyleMedium1`–`28` or `TableStyleDark1`–`11`, or `null` for none; default `TableStyleMedium2`) and + the banding flags `ShowRowStripes`, `ShowColumnStripes`, `ShowFirstColumn`, `ShowLastColumn`. +- With `FirstColumn > 0`, rows you write yourself must `Skip(FirstColumn)` to reach the table; + `WriteTableAsync` does it for you. +- One table can be open per sheet at a time; write as many as you like one after another. Table names + must be valid Excel names, must not read as cell references (`A1`, `R1C1`), and must be unique in the + workbook, ignoring case. A table with no data rows gets one empty row, as Excel requires. +- Totals rows, custom table styles, and tables in XLS or CSV are not supported. + ## Prefetch compression (XLSX/XLSB writing) The write-side mirror of [Prefetch decompression](reading.md#prefetch-decompression-xlsxxlsb). XLSX and XLSB @@ -195,6 +241,7 @@ and by default that happens on the calling thread. Set `PrefetchWrite` on the wr onto a background thread, so the caller keeps building the next batch of rows while the previous one compresses: + ```csharp await using var wb = XlsxWorkbookWriter.Create(stream, leaveOpen: true, options: new XlsxWriterOptions { PrefetchWrite = true }); XlsxSheetWriter sheet = wb.AddSheet("S1"); @@ -209,7 +256,7 @@ foreach (var record in records) await sheet.EndAsync(); ``` -`XlsbWriterOptions` has the same property. XLS and CSV are uncompressed, so +[`XlsbWriterOptions`](xref:ExcelReader.Core.Writer.Xlsb.XlsbWriterOptions) has the same property. XLS and CSV are uncompressed, so they have nothing to overlap and do not offer it. Writing a 50,000-row workbook (`WriteBenchmark`, both figures from one run): diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..be2d4c13 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,5 @@ +--- +_layout: landing +--- + +[!INCLUDE [README](../README.md)] diff --git a/docs/performance/benchmarks.md b/docs/performance/benchmarks.md index e75b68cc..665a46fd 100644 --- a/docs/performance/benchmarks.md +++ b/docs/performance/benchmarks.md @@ -15,32 +15,32 @@ Compares ExcelReader against established XLSX libraries on the same generated wo | Scenario | ExcelReader | Sylvan | OfficeIMO | SpreadCheetah | |---|---:|---:|---:|---:| -| Cell-by-cell read | 7.207 ms, 3.82 KB | 35.203 ms, 1.89 MB | 104.726 ms, 6.84 MB | - | -| Cell-by-cell read async | 7.953 ms, 3.89 KB | - | - | - | -| Typed row parsing | 10.935 ms, 3.87 MB | 56.855 ms, 10.47 MB | 104.530 ms, 5.69 MB | - | -| Typed row parsing async | 11.241 ms, 3.87 MB | 60.501 ms, 10.48 MB | - | - | -| Typed row parsing, shared strings | 10.849 ms, 2.29 MB | - | - | - | +| Cell-by-cell read | 6.847 ms, 3.57 KB | 36.131 ms, 1.89 MB | 106.861 ms, 6.84 MB | - | +| Cell-by-cell read async | 7.391 ms, 3.64 KB | - | - | - | +| Typed row parsing | 10.668 ms, 3.87 MB | 56.150 ms, 10.48 MB | 107.805 ms, 5.70 MB | - | +| Typed row parsing async | 10.833 ms, 3.87 MB | 59.036 ms, 10.48 MB | - | - | +| Typed row parsing, shared strings | 9.216 ms, 2.29 MB | - | - | - | | Workbook writing | 10.913 ms, 4.02 MB | - | 18.166 ms, 4.02 MB | 15.128 ms, 15.84 MB | | Workbook writing, shared strings | 10.848 ms, 4.06 MB | - | - | - | -ExcelReader is ~4.9x faster than Sylvan for raw XLSX reads, allocating ~508x less, and ~14.5x faster than OfficeIMO. For typed parsing it is ~5.2x faster than Sylvan (~5.4x async against async). For XLSX writing, ExcelReader is ~1.4x faster than SpreadCheetah while allocating ~3.9x less memory, and ~1.7x faster than OfficeIMO. +ExcelReader is ~5.3x faster than Sylvan for raw XLSX reads, allocating ~543x less, and ~15.6x faster than OfficeIMO. For typed parsing it is ~5.3x faster than Sylvan (~5.4x async against async). For XLSX writing, ExcelReader is ~1.4x faster than SpreadCheetah while allocating ~3.9x less memory, and ~1.7x faster than OfficeIMO. OfficeIMO.Excel's typed parsing is hand-mapped from its `OpenDataReader`, because `RowsAs` needs an `r` attribute on every row and cell, which the spec makes optional and ExcelReader's writer omits. -Reading a shared-strings XLSX workbook with typed parsing is ~1% faster than the inline-string sheet above (10.849 ms vs. 10.935 ms) and allocates ~41% less (2.29 MB vs. 3.87 MB) — each distinct string decodes once into the shared-string cache instead of once per cell occurrence. +Reading a shared-strings XLSX workbook with typed parsing is ~14% faster than the inline-string sheet above (9.216 ms vs. 10.668 ms) and allocates ~41% less (2.29 MB vs. 3.87 MB) — each distinct string decodes once into the shared-string cache instead of once per cell occurrence. ### XLSB (BIFF12) | Scenario | ExcelReader | OfficeIMO | |---|---:|---:| -| Cell-by-cell read | 5.178 ms, 4.13 KB | 10.406 ms, 6.70 MB | -| Cell-by-cell read async | 5.249 ms, 4.20 KB | - | -| Typed row parsing | 8.211 ms, 3.87 MB | - | -| Typed row parsing async | 8.261 ms, 3.87 MB | - | +| Cell-by-cell read | 4.798 ms, 3.82 KB | 11.123 ms, 6.70 MB | +| Cell-by-cell read async | 4.845 ms, 3.89 KB | - | +| Typed row parsing | 7.819 ms, 3.87 MB | - | +| Typed row parsing async | 8.061 ms, 3.87 MB | - | | Workbook writing | 7.478 ms, 4.02 MB | - | | Workbook writing, shared strings | 7.370 ms, 4.06 MB | - | -XLSB is the fastest generated Excel format in these results: raw reads are ~1.4x faster than XLSX reads, typed parsing is ~1.3x faster than XLSX parsing, and writing is ~1.5x faster than XLSX writing. Against OfficeIMO's XLSB reader, ExcelReader is ~2.0x faster while allocating ~1,660x less. The XLSB writer is also ~2.0x faster than SpreadCheetah's XLSX writer on this benchmark while allocating ~75% less memory. +XLSB is the fastest generated Excel format in these results: raw reads are ~1.4x faster than XLSX reads, typed parsing is ~1.4x faster than XLSX parsing, and writing is ~1.5x faster than XLSX writing. Against OfficeIMO's XLSB reader, ExcelReader is ~2.3x faster while allocating ~1,800x less. The XLSB writer is also ~2.0x faster than SpreadCheetah's XLSX writer on this benchmark while allocating ~75% less memory. ### XLS (BIFF8) @@ -66,7 +66,7 @@ For raw CSV reads, all three libraries read text through spans. ExcelReader is ~ ### Parallel CSV -`CsvParallel.ParseAsync` and `CsvParallel.AggregateAsync` on generated files (8,000,000 rows of three ints for the narrow corpus, 4,300,000 rows of two strings, a date, two decimals and an int for the conversion-heavy one), by `degreeOfParallelism`. The machine has 8 physical / 16 logical cores. +`SheetParallel.ParseAsync` and `SheetParallel.AggregateAsync` over a `CsvSheet`, on generated files (8,000,000 rows of three ints for the narrow corpus, 4,300,000 rows of two strings, a date, two decimals and an int for the conversion-heavy one), by `degreeOfParallelism`. The machine has 8 physical / 16 logical cores. | Dop | Conversion-heavy, typed | Narrow ints, typed | Conversion-heavy, `ref struct` aggregate | Narrow ints, `ref struct` aggregate | |---:|---:|---:|---:|---:| @@ -76,6 +76,8 @@ For raw CSV reads, all three libraries read text through spans. ExcelReader is ~ | 8 | 285.2 ms, 680.15 MB | 204.7 ms, 254.63 MB | 158.3 ms, 1.54 MB | 94.6 ms, 1.62 MB | | 16 | 324.9 ms, 680.32 MB | 189.0 ms, 255.15 MB | 114.4 ms, 1.66 MB | 73.0 ms, 1.69 MB | +These numbers were measured before the rename to `SheetParallel`; the rename did not touch the CSV engine, and a scratch harness comparing the two builds on a 156 MB CSV showed no regression. + The typed path scales ~2.8x on the conversion-heavy corpus (bimodal at dop 16: 86.7 ms standard deviation, median 275.8 ms) and ~2.7x on narrow ints, where it flattens out after dop 4 (217.2 ms vs. 189.0 ms at dop 16). Its ~680 MB is one materialized row object per record, and at dop 16 that costs 42,000 Gen0 plus 41,000 Gen1 collections. The aggregate columns fold each record into a per-partition accumulator through a `ref struct` model, so text columns stay as `ReadOnlySpan` and no row object is ever materialized. They scale better than the typed path — ~6.9x (conversion-heavy) and ~6.5x (narrow) at dop 16 — because they are not fighting the allocator: at dop 16 the conversion-heavy aggregate is ~2.8x faster than typed while allocating ~410x less (1.66 MB against 680.32 MB), with **zero** garbage collections at every degree. The remaining allocation is per-partition bookkeeping, not per-row. @@ -91,19 +93,32 @@ Against [Sep](https://github.com/nietras/Sep)'s own `ParallelEnumerate` (all cor Sequentially, ExcelReader's typed parser is ~1.1x faster than Sep on narrow ints (490.6 ms vs. 521.7 ms), and ~1.3x faster on the conversion-heavy corpus. In parallel ExcelReader is ~2.3x (narrow) and ~3.3x (conversion-heavy) faster than Sep's, while yielding rows in file order. +### Parallel sheet reading + +`SheetParallel.AggregateAsync` over the first sheet of the real-data workbook (65,535 rows, 14 columns; XLSX 6.7 MB compressed, 34.4 MB sheet XML; XLSB 3.7 MB, 15.9 MB sheet records), folding every row of a typed model into a per-partition sum (`SheetParallelBenchmark`). One thread inflates the sheet and cuts it into row-aligned chunks; `Dop - 1` workers parse them. + +| Dop | XLSX | XLSB | +|---:|---:|---:| +| 1 | 52.60 ms, 8.02 MB | 35.77 ms, 8.02 MB | +| 2 | 36.30 ms, 8.13 MB | 24.48 ms, 8.07 MB | +| 4 | 16.89 ms, 8.11 MB | 13.32 ms, 8.07 MB | +| 8 | 17.06 ms, 8.16 MB | 13.67 ms, 8.09 MB | + +At dop 4 XLSX reads ~3.1x faster than sequentially and XLSB ~2.7x. Dop 2 is a single parse worker running alongside the inflating thread. Past dop 4 nothing improves: the one inflating thread is the floor (~14 ms for XLSX, ~10 ms for XLSB, see [Inflate](#inflate), plus the XLSB record walk that finds row boundaries). Allocation barely moves with dop (8.02 to 8.16 MB): it is the typed model's strings. + ### Real data reads This benchmark reads a real workbook exported in multiple formats. | Format | ExcelReader | ExcelReader, prefetch | Sylvan | |---|---:|---:|---:| -| XLSX | 49.747 ms, 7.62 KB | 27.818 ms, 25.19 KB | 193.951 ms, 644.23 KB | -| XLSM | 50.311 ms, 7.62 KB | 28.100 ms, 25.01 KB | 197.515 ms, 644.30 KB | -| XLSB | 27.658 ms, 7.18 KB | 16.899 ms, 15.70 KB | 30.150 ms, 338.54 KB | -| XLS | 10.919 ms, 12.02 KB | n/a | 18.306 ms, 185.90 KB | -| CSV | 4.955 ms, 848 B | n/a | 4.491 ms, 40.26 KB | +| XLSX | 41.275 ms, 7.31 KB | 29.616 ms, 23.52 KB | 197.561 ms, 644.23 KB | +| XLSM | 43.649 ms, 7.31 KB | 29.347 ms, 23.48 KB | 194.102 ms, 644.30 KB | +| XLSB | 25.303 ms, 6.87 KB | 17.588 ms, 15.20 KB | 29.771 ms, 338.54 KB | +| XLS | 11.280 ms, 12.02 KB | n/a | 18.379 ms, 185.90 KB | +| CSV | 5.072 ms, 888 B | n/a | 4.585 ms, 40.26 KB | -On this real-data workload, ExcelReader is ~3.9x faster than Sylvan for XLSX and ~3.9x for XLSM, essentially tied for XLSB (~1.09x), and ~1.7x faster for XLS — allocating ~84x less for XLSX and XLSM, ~47x less for XLSB and ~15x less for XLS. On CSV, where both sides read text through spans, Sylvan.Data.Csv is ~10% faster, while ExcelReader allocates ~49x less. The prefetch column is the opt-in [`PrefetchDecompression`](../guide/reading.md#prefetch-decompression-xlsxxlsb) option; XLS and CSV are uncompressed, so it does not apply to them. +On this real-data workload, ExcelReader is ~4.8x faster than Sylvan for XLSX and ~4.4x for XLSM, ~1.2x faster for XLSB, and ~1.6x faster for XLS — allocating ~88x less for XLSX and XLSM, ~49x less for XLSB and ~15x less for XLS. On CSV, where both sides read text through spans, Sylvan.Data.Csv is ~11% faster, while ExcelReader allocates ~46x less. The prefetch column is the opt-in [`PrefetchDecompression`](../guide/reading.md#prefetch-decompression-xlsxxlsb) option; XLS and CSV are uncompressed, so it does not apply to them. ### In-memory real-data reads @@ -111,18 +126,18 @@ The real-data benchmark also measures the in-memory path for workbook content lo | Method | Mean | StdDev | Allocated | |---|---:|---:|---:| -| Xlsx_ExcelReader_Memory | 49.968 ms | 0.156 ms | 7.38 KB | -| Xlsx_ExcelReader_Memory_Prefetch | 28.667 ms | 0.090 ms | 25.19 KB | -| Xlsm_ExcelReader_Memory | 50.871 ms | 0.172 ms | 7.38 KB | -| Xlsm_ExcelReader_Memory_Prefetch | 28.844 ms | 0.081 ms | 25.07 KB | -| Xlsb_ExcelReader_Memory | 29.321 ms | 0.070 ms | 8.88 KB | -| Xlsb_ExcelReader_Memory_Prefetch | 15.741 ms | 0.027 ms | 19.11 KB | -| Xls_ExcelReader_Memory | 10.514 ms | 0.033 ms | 11.88 KB | -| Csv_ExcelReader_Memory | 4.745 ms | 0.004 ms | 576 B | +| Xlsx_ExcelReader_Memory | 42.521 ms | 0.905 ms | 7.05 KB | +| Xlsx_ExcelReader_Memory_Prefetch | 29.167 ms | 0.353 ms | 23.03 KB | +| Xlsm_ExcelReader_Memory | 40.925 ms | 0.140 ms | 7.05 KB | +| Xlsm_ExcelReader_Memory_Prefetch | 30.345 ms | 1.268 ms | 23.22 KB | +| Xlsb_ExcelReader_Memory | 27.084 ms | 1.294 ms | 8.55 KB | +| Xlsb_ExcelReader_Memory_Prefetch | 18.140 ms | 0.703 ms | 16.79 KB | +| Xls_ExcelReader_Memory | 10.704 ms | 0.139 ms | 11.88 KB | +| Csv_ExcelReader_Memory | 4.759 ms | 0.022 ms | 616 B | -`Csv_ExcelReader_Memory` is both faster (4.745 ms vs. 4.955 ms for `Csv_ExcelReader`) and allocates less (576 B vs. 848 B) than its stream twin. +`Csv_ExcelReader_Memory` is both faster (4.759 ms vs. 5.072 ms for `Csv_ExcelReader`) and allocates less (616 B vs. 888 B) than its stream twin. -`Xls_ExcelReader_Memory` is ~4% faster than its stream twin (10.514 ms vs. 10.919 ms for `Xls_ExcelReader`), with near-identical allocation (11.88 KB vs. 12.02 KB). In earlier runs the in-memory path was ~33% faster, thanks to `BiffCursor` caching the enclosing contiguous sector run; the stream path has since caught up. +`Xls_ExcelReader_Memory` is ~5% faster than its stream twin (10.704 ms vs. 11.280 ms for `Xls_ExcelReader`), with near-identical allocation (11.88 KB vs. 12.02 KB). In earlier runs the in-memory path was ~33% faster, thanks to `BiffCursor` caching the enclosing contiguous sector run; the stream path has since mostly caught up. ### String-heavy reads @@ -130,22 +145,34 @@ The real-data corpus above is mostly numbers and dates — its shared-string tab | Format | ExcelReader | ExcelReader, prefetch | Sylvan | |---|---:|---:|---:| -| XLSX | 41.18 ms, 2.18 MB | 27.36 ms, 2.27 MB | 178.73 ms, 17.41 MB | -| XLSB | 39.48 ms, 2.18 MB | 26.04 ms, 2.26 MB | 62.55 ms, 17.38 MB | +| XLSX | 32.84 ms, 2.18 MB | 24.03 ms, 2.21 MB | 186.05 ms, 17.41 MB | +| XLSB | 32.44 ms, 2.18 MB | 21.12 ms, 2.20 MB | 65.22 ms, 17.38 MB | -Both formats handle this well: ~4.3x and ~1.6x faster than Sylvan for XLSX and XLSB respectively, at roughly ~8x less memory in both cases, with no garbage collections in either configuration. XLSB's shared-string path previously materialized its table eagerly (~27 MB here); `ParseSharedStreaming` brought it in line with the XLSX streaming/pooling path, cutting allocation by ~12x on this workload. +Both formats handle this well: ~5.7x and ~2.0x faster than Sylvan for XLSX and XLSB respectively, at roughly ~8x less memory in both cases, with no garbage collections in either configuration. XLSB's shared-string path previously materialized its table eagerly (~27 MB here); `ParseSharedStreaming` brought it in line with the XLSX streaming/pooling path, cutting allocation by ~12x on this workload. `XlsxSharedStringHotPathBenchmark` splits the XLSX read into stages on the same workbook. The stored rows re-zip it without compression, so they show what parsing costs once inflate is gone: | Stage | Mean | Allocated | |---|---:|---:| -| Whole read | 41.17 ms | 2.18 MB | -| Whole read, prefetch | 28.21 ms | 2.27 MB | -| Shared-string table only (open + first row) | 13.54 ms | 747 KB | -| Whole read, stored | 19.82 ms | 2.18 MB | -| Shared-string table only, stored | 3.11 ms | 746 KB | +| Whole read | 33.87 ms | 2.18 MB | +| Whole read, prefetch | 23.36 ms | 2.21 MB | +| Shared-string table only (open + first row) | 9.31 ms | 747 KB | +| Whole read, stored | 20.23 ms | 2.18 MB | +| Shared-string table only, stored | 3.42 ms | 746 KB | + +About 40% of the default read is inflate: without it the whole read drops from 33.87 ms to 20.23 ms, and the shared-string table from 9.31 ms to 3.42 ms. What remains, ~17 ms, is the worksheet scan. + +### Inflate + +`InflateBenchmark` decompresses one ZIP entry per method with `System.IO.Compression.DeflateStream` and with the library's managed `InflateStream`, which every ZIP-based reader now uses: + +| Entry | `DeflateStream` | `InflateStream` | Ratio | +|---|---:|---:|---:| +| XLSX `sheet1.xml` (6.7 MB → 34.4 MB) | 23.789 ms, 280 B | 13.741 ms, 200 B | 0.58 | +| XLSB `sheet1.bin` (3.7 MB → 15.9 MB) | 13.143 ms, 280 B | 9.734 ms, 200 B | 0.74 | +| String-heavy `sharedStrings.xml` | 10.571 ms, 281 B | 6.106 ms, 200 B | 0.58 | -About half of the default read is inflate: without it the whole read drops from 41.17 ms to 19.82 ms, and the shared-string table from 13.54 ms to 3.11 ms. What remains, ~17 ms, is the worksheet scan. +Without prefetch a read is inflate plus parse, so the faster inflate converts directly; with prefetch the two overlap and the read is bounded by the slower stage, which is why the prefetch columns above moved little. ### Matched-work reads (`*_Materialized`) @@ -157,30 +184,30 @@ Real-data workbook, per format: | Format | Mean | StdDev | Allocated | |---|---:|---:|---:| -| XLSX | 52.42 ms | 0.05 ms | 17.09 KB | -| XLSM | 51.33 ms | 0.07 ms | 17.09 KB | -| XLSB | 30.97 ms | 0.09 ms | 16.65 KB | -| XLS | 12.62 ms | 0.03 ms | 21.48 KB | -| CSV | 18.42 ms | 0.09 ms | 35.71 MB | +| XLSX | 43.03 ms | 0.15 ms | 16.78 KB | +| XLSM | 43.64 ms | 0.28 ms | 16.78 KB | +| XLSB | 28.01 ms | 0.90 ms | 16.34 KB | +| XLS | 12.95 ms | 0.25 ms | 21.48 KB | +| CSV | 19.31 ms | 0.22 ms | 35.71 MB | Generated 50,000-row XLSX workbook: | Scenario | Mean | StdDev | Allocated | |---|---:|---:|---:| -| Cell-by-cell read, materialized | 8.41 ms | 0.02 ms | 1.58 MB | +| Cell-by-cell read, materialized | 7.89 ms | 0.03 ms | 1.58 MB | String-heavy workbook (65,536 rows, ~190,000 distinct shared strings): | Format | Mean | StdDev | Gen0 | Gen1 | Gen2 | Allocated | |---|---:|---:|---:|---:|---:|---:| -| XLSX | 61.82 ms | 0.83 ms | 1,111.1 | 1,000.0 | 333.3 | 15.26 MB | -| XLSB | 59.38 ms | 0.92 ms | 1,111.1 | 1,000.0 | 333.3 | 15.26 MB | +| XLSX | 55.51 ms | 0.58 ms | 1,111.1 | 1,000.0 | 333.3 | 15.25 MB | +| XLSB | 55.55 ms | 0.95 ms | 1,111.1 | 1,000.0 | 333.3 | 15.25 MB | Reading the allocation columns against the tables above gives the honest shape of the tradeoff: -- **CSV real data:** 35.71 MB materialized against 848 B for the span-based read of the same file. Every CSV field is a distinct string, so nothing dedupes — this is where zero-copy reading earns its keep outright. -- **XLSB real data:** 16.65 KB, essentially cheap. That corpus repeats a small set of values, so the shared-string table dedupes and the reader's string cache materializes each distinct value once. -- **String-heavy XLSX:** 15.26 MB materialized, against **Sylvan's 17.41 MB on the same workload** — doing matched work here, ExcelReader allocates ~12% *less* than Sylvan, with 333 Gen2 collections, and is still ~2.9x faster (61.82 ms vs. 178.73 ms). The same allocation holds for XLSB (15.26 MB vs. Sylvan's 17.38 MB, also ~12% less, 333 Gen2 collections), where matched-work time is ~5% ahead of Sylvan (59.38 ms vs. 62.55 ms). This was previously an inversion (ExcelReader allocated ~1.75x *more* than Sylvan here): the per-reader shared-string dedup cache was an unpresized `Dictionary`, and at ~190,000 distinct values its resize/rehash churn (several of the largest resizes landing on the LOH) accounted for the entire gap — the strings themselves were never the problem, since both readers retain the same ~190,000 distinct instances. Replacing the dictionary with a `string?[]` indexed by shared-string index (sized exactly from the table's known count, no resizing) cut the allocation in half and cut wall-clock time by 14-16% on this benchmark too, since the churn was costing cycles, not just memory. +- **CSV real data:** 35.71 MB materialized against 888 B for the span-based read of the same file. Every CSV field is a distinct string, so nothing dedupes — this is where zero-copy reading earns its keep outright. +- **XLSB real data:** 16.34 KB, essentially cheap. That corpus repeats a small set of values, so the shared-string table dedupes and the reader's string cache materializes each distinct value once. +- **String-heavy XLSX:** 15.25 MB materialized, against **Sylvan's 17.41 MB on the same workload** — doing matched work here, ExcelReader allocates ~12% *less* than Sylvan, with 333 Gen2 collections, and is still ~3.4x faster (55.51 ms vs. 186.05 ms). The same allocation holds for XLSB (15.25 MB vs. Sylvan's 17.38 MB, also ~12% less, 333 Gen2 collections), where matched-work time is ~17% ahead of Sylvan (55.55 ms vs. 65.22 ms). This was previously an inversion (ExcelReader allocated ~1.75x *more* than Sylvan here): the per-reader shared-string dedup cache was an unpresized `Dictionary`, and at ~190,000 distinct values its resize/rehash churn (several of the largest resizes landing on the LOH) accounted for the entire gap — the strings themselves were never the problem, since both readers retain the same ~190,000 distinct instances. Replacing the dictionary with a `string?[]` indexed by shared-string index (sized exactly from the table's known count, no resizing) cut the allocation in half and cut wall-clock time by 14-16% on this benchmark too, since the churn was costing cycles, not just memory. The takeaway is not that one column beats the other: it is that ExcelReader's headline read numbers come from a zero-copy path competitors do not expose, and when it does the same work as them, the gap narrows — and here, with the dedup cache fixed, no longer inverts even at high shared-string cardinality. @@ -216,25 +243,25 @@ A `ref struct` model (see [Parse into a ref struct](../guide/parsing.md#parse-in | Target | Mean | Allocated | |---|---:|---:| -| `class` (`ExcelParser`) | 10.94 ms | 3.87 MB | -| `struct` (`ExcelParser`) | 9.93 ms | 1.58 MB | -| `ref struct` + span binding (`ExcelParser`) | 9.30 ms | 4.36 KB | +| `class` (`ExcelParser`) | 11.22 ms | 3.87 MB | +| `struct` (`ExcelParser`) | 10.11 ms | 1.58 MB | +| `ref struct` + span binding (`ExcelParser`) | 9.65 ms | 4.12 KB | -Parsing into a `ref struct` with a `ReadOnlySpan` text column removes essentially all per-row allocation — ~99.9% less than the `class` baseline — and is ~6% faster than the `struct` target, since there's no per-row `string` allocation for the text column. It is not AOT/trim-safe (reflection-based, same tradeoff as `ExcelParser.FromAttributes`). It can be consumed with `foreach` or `await foreach` but not through `IEnumerable`/`IAsyncEnumerable`/LINQ — a `ref struct` element can't be boxed through those interfaces. +Parsing into a `ref struct` with a `ReadOnlySpan` text column removes essentially all per-row allocation — ~99.9% less than the `class` baseline — and is ~5% faster than the `struct` target, since there's no per-row `string` allocation for the text column. It is not AOT/trim-safe (reflection-based, same tradeoff as `ExcelParser.FromAttributes`). It can be consumed with `foreach` or `await foreach` but not through `IEnumerable`/`IAsyncEnumerable`/LINQ — a `ref struct` element can't be boxed through those interfaces. ### ADO.NET bridge (`IDataReader`) -`DataReaderBenchmark`: the generated 50,000-row XLSX workbook (header plus four columns) read through [`ExcelDataReader`](../guide/reading.md#bridge-to-adonet-idatareader), against iterating the same rows directly. The reader is held as `IDataReader`, the way `SqlBulkCopy`, `DataTable.Load` and Dapper hold it. +`DataReaderBenchmark`: the generated 50,000-row XLSX workbook (header plus four columns) read through [`SheetDataReader`](../guide/reading.md#bridge-to-adonet-idatareader), against iterating the same rows directly. The reader is held as `IDataReader`, the way `SqlBulkCopy`, `DataTable.Load` and Dapper hold it. | Access | Mean | Ratio | Allocated | |---|---:|---:|---:| -| Raw rows (`row[i].Value`, baseline) | 6.878 ms | 1.00 | 3.87 KB | -| `GetBytes` (text column only) | 7.222 ms | 1.05 | 4.55 KB | -| `GetValue` (every column) | 9.660 ms | 1.40 | 5.01 MB | -| Typed getters (`GetString`/`GetInt32`/`GetDateTime`/`GetDouble`) | 9.059 ms | 1.32 | 1.58 MB | -| `DataTable.Load` | 63.615 ms | 9.25 | 22.84 MB | +| Raw rows (`row[i].Value`, baseline) | 6.845 ms | 1.00 | 3.62 KB | +| `GetBytes` (text column only) | 7.039 ms | 1.03 | 4.31 KB | +| `GetValue` (every column) | 9.479 ms | 1.39 | 5.01 MB | +| Typed getters (`GetString`/`GetInt32`/`GetDateTime`/`GetDouble`) | 9.094 ms | 1.33 | 1.58 MB | +| `DataTable.Load` | 60.509 ms | 8.85 | 22.84 MB | -The bridge costs ~1.3–1.4x over reading rows directly. `GetValue` boxes every value (the 5 MB), and the typed getters allocate only the text column's `string`s. `GetBytes` copies UTF-8 without making a `string`, so it stays within ~5% of the raw read. `DataTable.Load` is dominated by `DataTable`'s own row storage: the same values cost 9.660 ms to read through `GetValue` alone. +The bridge costs ~1.3–1.4x over reading rows directly. `GetValue` boxes every value (the 5 MB), and the typed getters allocate only the text column's `string`s. `GetBytes` copies UTF-8 without making a `string`, so it stays within ~3% of the raw read. `DataTable.Load` is dominated by `DataTable`'s own row storage: the same values cost 9.479 ms to read through `GetValue` alone. ### Arrow conversion @@ -255,16 +282,16 @@ On the same file, letting `Excel.InferSchema` guess the schema costs about the s | Scenario | Mean | Allocated | |---|---:|---:| -| Small, plain | 0.198 ms | 9.15 KB | -| Small, encrypted, stream | 26.129 ms | 38.39 KB | -| Small, encrypted, stream, `VerifyEncryptedIntegrity` | 25.641 ms | 40.54 KB | -| Small, encrypted, in memory | 26.001 ms | 53.53 KB | -| Small, encrypted, open only | 25.780 ms | 32.72 KB | -| 300,000 rows, plain | 39.180 ms | 3.87 KB | -| 300,000 rows, encrypted | 66.278 ms | 29.79 KB | -| 300,000 rows, encrypted, open only | 25.744 ms | 29.10 KB | +| Small, plain | 0.208 ms | 8.84 KB | +| Small, encrypted, stream | 25.880 ms | 38.07 KB | +| Small, encrypted, stream, `VerifyEncryptedIntegrity` | 26.150 ms | 40.16 KB | +| Small, encrypted, in memory | 26.600 ms | 53.24 KB | +| Small, encrypted, open only | 26.378 ms | 32.54 KB | +| 300,000 rows, plain | 38.283 ms | 3.62 KB | +| 300,000 rows, encrypted | 65.485 ms | 29.53 KB | +| 300,000 rows, encrypted, open only | 26.235 ms | 28.92 KB | -Nearly all of the cost is a fixed ~26 ms paid when the workbook is opened: deriving the key through 100,000 SHA-512 iterations, which the format requires so that passwords are slow to brute-force. It does not depend on file size (open-only is 25.78 ms for 15 KB and 25.74 ms for 300,000 rows). After that, decrypting while reading costs ~3% over a plain read: 66.28 ms − 25.74 ms = 40.53 ms, against 39.18 ms unencrypted. On the small fixture, integrity verification and the in-memory path cost nothing measurable. The fixture is too small to show the extra pass that verification makes over a large stream. +Nearly all of the cost is a fixed ~26 ms paid when the workbook is opened: deriving the key through 100,000 SHA-512 iterations, which the format requires so that passwords are slow to brute-force. It does not depend on file size (open-only is 26.38 ms for 15 KB and 26.24 ms for 300,000 rows). After that, decrypting while reading costs ~3% over a plain read: 65.49 ms − 26.24 ms = 39.25 ms, against 38.28 ms unencrypted. On the small fixture, integrity verification and the in-memory path stay within ~3% of the plain encrypted read, inside the spread of the key derivation itself. The fixture is too small to show the extra pass that verification makes over a large stream. ### Cold start @@ -299,25 +326,25 @@ dotnet run --project tests/ExcelReader.Benchmarks/ExcelReader.Benchmarks.csproj | Entry point | Shape handed back | Mean | Managed allocated | |---|---|---:|---:| -| `xl_next_row` | one row per call, serialized into a caller buffer | 40.35 ms | 3.71 MB | -| `xl_read_all_blob` | whole sheet in one buffer | 43.59 ms | 21.06 MB | -| `xl_read_all_decoded` | whole sheet as native row structs | 54.56 ms | 5.71 MB | -| `xl_parse_typed` | typed native columns | 40.79 ms | 11.61 MB | -| `xl_parse_arrow` | Arrow C Data Interface arrays | 39.51 ms | 11.62 MB | +| `xl_rows_next` | one row per call, serialized into a caller buffer | 37.93 ms | 3.71 MB | +| `xl_rows_read_all_blob` | whole sheet in one buffer | 42.74 ms | 21.06 MB | +| `xl_rows_read_all_decoded` | whole sheet as native row structs | 56.73 ms | 5.71 MB | +| `xl_parse_typed` | typed native columns | 35.73 ms | 3.71 MB | +| `xl_parse_arrow` | Arrow C Data Interface arrays | 36.93 ms | 3.72 MB | -The allocation column is **managed memory only**. `MemoryDiagnoser` cannot see the native blocks that `xl_read_all_decoded`, `xl_parse_typed` and `xl_parse_arrow` return their data in. +The allocation column is **managed memory only**. `MemoryDiagnoser` cannot see the native blocks that `xl_rows_read_all_decoded`, `xl_parse_typed` and `xl_parse_arrow` return their data in. -Row by row, `xl_next_row` costs ~1.4x the managed in-memory read of the same file (40.35 ms vs. 29.321 ms for `Xlsb_ExcelReader_Memory` under [In-memory real-data reads](#in-memory-real-data-reads); different benchmark class, same run). Every cell crosses the ABI as text, so each binary XLSB number is formatted to UTF-8 on the way out. `xl_read_all_blob` adds ~8% over it, because the whole sheet is buffered before it is copied out. `xl_read_all_decoded` adds ~35%, because it allocates a native block per row. `xl_parse_typed` and `xl_parse_arrow` cost about the same as a row-by-row read but return columnar native buffers with numbers still binary, so they are the path to use for bulk typed loads. +Row by row, `xl_rows_next` costs ~1.4x the managed in-memory read of the same file (37.93 ms vs. 27.084 ms for `Xlsb_ExcelReader_Memory` under [In-memory real-data reads](#in-memory-real-data-reads)). Every cell crosses the ABI as text, so each binary XLSB number is formatted to UTF-8 on the way out. `xl_rows_read_all_blob` adds ~13% over it, because the whole sheet is buffered before it is copied out. `xl_rows_read_all_decoded` adds ~50%, because it allocates a native block per row. `xl_parse_typed` and `xl_parse_arrow` are slightly faster than a row-by-row read (~3–6%) and return columnar native buffers with numbers still binary, so they are the path to use for bulk typed loads. `ChunkedParseBenchmark` compares `xl_parse_typed` on a whole sheet against draining the same sheet in batches through `xl_typed_reader_open`/`xl_typed_reader_next`. It uses the generated 50,000-row XLSX file, read from disk, with three nullable columns (string, int64, float64): | Mode | Mean | Managed allocated | |---|---:|---:| -| Whole sheet (`xl_parse_typed`) | 8.73 ms | 1.29 MB | -| Batches of 10,000 rows | 8.49 ms | 1.59 MB | -| Batches of 1,000 rows | 8.47 ms | 1.51 MB | +| Whole sheet (`xl_parse_typed`) | 8.93 ms | 10 KB | +| Batches of 10,000 rows | 8.36 ms | 16.95 KB | +| Batches of 1,000 rows | 8.53 ms | 81.29 KB | -Batching costs nothing in time: both batch sizes land within ~3% of one whole-sheet table. A caller can bound its native memory to one batch at a time without paying for it. +Batching costs nothing in time: both batch sizes are slightly faster than one whole-sheet table (~4–6%). A caller can bound its native memory to one batch at a time without paying for it. ### C++ and Rust bindings @@ -332,24 +359,24 @@ zero-copy advantage the other can't take: | Comparison | Format | ExcelReader | Competitor | Ratio | |---|---|---:|---:|---:| -| vs. [calamine](https://github.com/tafia/calamine) (Rust) | XLSX | 85.8 ms | 272.0 ms | ~3.2x faster | -| vs. calamine (Rust) | XLSB | 53.0 ms | 86.8 ms | ~1.6x faster | -| vs. [DuckDB](https://github.com/duckdb/duckdb) `read_xlsx` (C++) | XLSX | 73.7 ms | 433.6 ms | ~5.9x faster | -| vs. [xlsxio](https://github.com/brechtsanders/xlsxio) (C) | XLSX | 73.7 ms | 505.0 ms | ~6.9x faster | -| vs. [xlnt](https://github.com/tfussell/xlnt) (C++) | XLSX | 73.7 ms | 2,494.2 ms | ~34x faster | -| vs. [zsv](https://github.com/liquidaty/zsv) `fast` engine (C) | CSV | 30.4 ms | 22.7 ms | ~1.3x **slower** | +| vs. [calamine](https://github.com/tafia/calamine) (Rust) | XLSX | 79.1 ms | 279.7 ms | ~3.5x faster | +| vs. calamine (Rust) | XLSB | 46.9 ms | 87.5 ms | ~1.9x faster | +| vs. [DuckDB](https://github.com/duckdb/duckdb) `read_xlsx` (C++) | XLSX | 62.9 ms | 410.8 ms | ~6.5x faster | +| vs. [xlsxio](https://github.com/brechtsanders/xlsxio) (C) | XLSX | 62.9 ms | 512.9 ms | ~8.2x faster | +| vs. [xlnt](https://github.com/tfussell/xlnt) (C++) | XLSX | 62.9 ms | 2,394.1 ms | ~38x faster | +| vs. [zsv](https://github.com/liquidaty/zsv) `fast` engine (C) | CSV | 27.2 ms | 20.4 ms | ~1.3x **slower** | Writing the same 14 columns × 65,535 rows from row-shaped data, the C++ suite measures -`write_sheet` at 97.8–100.1 ms against DuckDB's 833–870 ms (~8.5–8.7x), libxlsxwriter's 1,184 ms (~12x), -xlsxio's 2,454 ms (~25x) and xlnt's 5,641 ms (~56x). The Rust suite measures 53.3 ms against -rust_xlsxwriter's 326.9 ms (~6.1x) over 7 columns. Caveats for both are in the binding READMEs. +`write_sheet` at 95.5–96.5 ms against DuckDB's 829–833 ms (~8.6–8.7x), libxlsxwriter's 1,194 ms (~12x), +xlsxio's 2,433 ms (~25x) and xlnt's 5,436 ms (~57x). The Rust suite measures 51.9 ms against +rust_xlsxwriter's 328.3 ms (~6.3x) over 7 columns. Caveats for both are in the binding READMEs. DuckDB, xlsxio and xlnt do not read `.xlsb`, so those comparisons are XLSX-only. calamine is a fast, well-optimized reader in its own right — the gap there is real but not the order of magnitude seen against the C/C++ competitors. zsv is the one competitor ahead. The CSV row is the typed workload, where both sides build the same -14-column rows with owned strings. Just counting cell bytes, zsv is ~4x faster, and that gap is +14-column rows with owned strings. Just counting cell bytes, zsv is ~3.3x faster, and that gap is ExcelReader's C ABI, which serializes every row into a buffer, not its CSV parser. That suite is built with GCC (zsv does not compile under MSVC). Its caveats are in the C++ README. diff --git a/docs/performance/reader-performance.md b/docs/performance/reader-performance.md index 328a2f1a..6a77667c 100644 --- a/docs/performance/reader-performance.md +++ b/docs/performance/reader-performance.md @@ -488,9 +488,12 @@ starts. Typed models, allocating, own interleave: | dop 6 | 5.068 ms (2.80x) | | dop 12 | 4.107 ms (3.46x) | -This is the one place in the library where parallelism converts, and it converts because CSV has no -inflate floor. It is only available on the typed path: `Row` and `Cell` are `ref struct`s and cannot -cross a thread boundary, so raw row enumeration hits the same wall as XLSX. +This was the one place in the library where parallelism converted, and it converted because CSV has no +inflate floor. Since v7 `SheetParallel` also reads XLSX and XLSB on several threads (see "Parallel intra-sheet +parsing" below), with a smaller gain because inflate is still sequential. Parallel reading is only available on +the typed path: `Row` and `Cell` are `ref struct`s and cannot cross a thread boundary, so raw row enumeration +hits the same wall. `CsvParallel` is named `SheetParallel` in v7; the entries in this file keep the names they +were measured under. ## Approaches measured and rejected @@ -524,6 +527,37 @@ because inflate is then all that is left. It would also only ever serve the typed-parsing layer: `Row`/`Cell` are `ref struct` and cannot cross a thread boundary. +**Update 2026-10-10: built.** `InflateStream` moved the inflate floor, so the prefetched XLSX read became +parse-bound and the reasoning above no longer holds. A spike (Ryzen 7 5700X, real-data 65K corpus, min of 30 +interleaved rounds, sheet stage only, outputs identical to sequential) measured: + +| | XLSX (6.7 → 34.4 MB) | XLSB (3.7 → 15.9 MB) | +|---|---|---| +| read today, no prefetch / prefetch | 44.6 / 29.2 ms | 26.5 / 15.8 ms | +| inflate only, whole buffer | 14.9 ms | 9.5 ms | +| parse only, dop 1 / 2 / 4 / 8 | 29.0 / 14.8 / 7.8 / 4.2 ms | 15.4 / 7.8 / 4.0 / 2.3 ms | +| A: 1 inflate thread + N parse workers (N ≥ 2) | 16.0 ms (1.8x) | 12.5 ms (1.25x) | +| B oracle: parallel inflate + parse, 8 threads | 7.7 ms | 3.6 ms | + +A is what shipped: `SheetParallel` over XLSX and XLSB, one producer that inflates and cuts row-aligned chunks, +`dop - 1` parse workers, an ordered merge (see ARCHITECTURE.md). The spike's consumer was light and its buffers +were pre-touched, so those numbers are the shape of the gain, not a promise. Measured on the built pipeline +(JIT harness, same corpus and method, light consumer, 256 KB chunks): + +| | XLSX | XLSB | +|---|---|---| +| prefetched sequential read | 28.35 ms | 15.65 ms | +| pipeline, dop 3 to 16 | 17.6 ms (1.61x) | 13.3 ms (1.18x) | +| typed `ParseAsync`, sequential / dop 4 | 55.4 / 18.9 ms | 37.6 / 14.4 ms | +| `xl_parse_typed` over the published NativeAOT DLL, dop 1 / dop 0 | 54.1 / 20.3 ms (2.6x) | 28.2 / 15.2 ms (1.85x) | + +Under the JIT the sheet stage stops scaling at dop 3, one producer plus two workers: dop 2 is a plain overlap of inflate and parse and gains +nothing. The chunk size defaults to 256 KB, down from 1 MB: it removes ~0.5 ms of serial tail, and 4 MB adds +~3.7 ms. XLSB stops at ~13 ms against a 9.6 ms inflate floor because the producer walks every BIFF12 record +(~2.2 ms) and `InflateStream.Read` adds a copy (~0.7 ms). Only parallel inflate moves that. The +BenchmarkDotNet tables for this are in `benchmarks.md`; they were taken on a loaded machine and are pending an +idle rerun. + ### Faster inflate — measured separately, no gain libdeflate was measured against `System.IO.Compression` on this workload before this investigation: @@ -531,8 +565,11 @@ no faster, and slower in places. libdeflate is whole-buffer only — no streamin the entire sheet to be materialized before parsing starts and destroys the inflate/parse overlap `PrefetchStream` exists to provide. -**The inflate floor does not move by swapping compression libraries.** Since XLSX and XLSB are both -bounded by it, this is the single most important negative result here. +**The inflate floor does not move by swapping in a whole-buffer compression library.** That is what +this measurement showed, and it was read too broadly: the decoder itself was never at the floor. +Measured on its own libdeflate is about twice as fast as `DeflateStream` on these entries; what cost +it the win end to end was having no streaming API. "Fourth round" below measures decoders in +isolation and replaces `DeflateStream` with a managed streaming one. ### Lazy number parsing — a trade, not a win @@ -982,26 +1019,199 @@ before publishing. The C++ binding's full XLSX read moved from 93.2 to ~85.4 ms with no native change, from the XLSX changes above. +### NativeAOT XLSX gap, re-checked (2026-10-09, Ryzen 7 5700X) + +The gap quoted under "NativeAOT, again" (JIT 74.1 ms, NativeAOT 84.0 ms) predates `InflateStream`, the +regenerated profile and the native column buffers, so it was re-measured first. The python binding +reads `65K_Records_Data.xlsx` with `parse_typed` in 58.0–58.5 ms from the shipped library. The same +`ReadApi.OpenMemory` + `TypedApi.ParseTyped` calls in a scratch console, JIT against NativeAOT built +with the library's settings (`OptimizationPreference=Speed`, `x86-64-v3`, the `.mibc`), min of 40 +rounds: + +| stage | JIT | NativeAOT | +|---|---|---| +| inflate `sheet1.xml` (`InflateStream`) | 16.0 ms | 16.0 ms | +| rows only, no prefetch | 39.1 ms | 43.5 ms | +| rows + `TryGetDouble` on every number | 42.8 ms | 47.9 ms | +| `xl_parse_typed` | 54.0 ms | 60.7 ms | + +So the gap is ~12% in the harness and ~7% against the shipped library. **Inflate is at parity**, so +the decode loop is not part of it. About two thirds is the worksheet scan and a third is the typed +conversions after it. + +**A first harness left out `OptimizationPreference=Speed`** and read 66.5 ms for NativeAOT: ILC's +default blended mode inlines much less. Check a harness's ILC settings against the project's before +trusting a gap measured in it. + +The disassembly, with `--codegenopt:JitDisasm=ParseRowInWindow` and `JitStdOutFile` given to ILC as +`IlcArg` items through a `CustomAfterMicrosoftCommonTargets` file: ILC's `ParseRowInWindow` is 846 +bytes with 15 inlinees and **a call to `ParseCellSpan`**. The JIT's Tier1 listing is 3,906 bytes with +202 inlinees and no such call; it pulled `ParseCellSpan` and its callees into the row loop. (Earlier +notes found `NoInlining` on `ParseCellSpan` costs the JIT 5–8%, so the inlined shape is the fast one.) +`[MethodImpl(MethodImplOptions.AggressiveInlining)]` on `ParseCellSpan` makes ILC do the same: + +| | NativeAOT before | after | JIT before | after | +|---|---|---|---|---| +| rows only | 43.5 ms | 41.8 ms | 39.2 ms | 39.5 ms | +| `xl_parse_typed` | 60.7 ms | 58.9 ms | 54.0 ms | 54.0 ms | + +That is 3–4% in the harness and neutral under the JIT. **It does not reach the shipped library**: the +real `ExcelReader.Native.dll` built with and without the attribute, alternated three times through the +python binding, measured 57.9–58.5 ms and 58.4–58.7 ms. The C++ `BM_ExcelReader_Xlsx_ParseOnly` agreed +(55.9–56.8 ms against 56.6–56.8). The real library's `ParseCellSpan` was a call in its baseline too, so +the attribute applies there; the Core-level harness just does not predict the end-to-end number, most +likely because the C ABI layer and the typed column writers dominate what the scan saves. Not kept. + +What is left is spread thin: ~0.7 ms in the consumer's double parse (`TryGetDouble` over every number +costs 4.4 ms under ILC and 3.7 ms under the JIT, with listings of about the same size, 662 and 641 +bytes), ~2.5 ms in the typed conversions, and the rest in the scan. No hot spot is visible in either +listing. + +### Fourth round (2026-10-08, i7-1365U): a managed inflate decoder + +Measured on the laptop, so only the ratios mean anything; the published tables need the Ryzen run. +Harnesses were throwaway console apps: interleaved rounds, minimum of 30 to 60, outputs compared +byte for byte. + +Inflate only, single thread pinned to one core, against `DeflateStream` (.NET 10): + +| decoder | XLSX `sheet1.xml`, 6.7 → 34.4 MB | XLSB `sheet1.bin`, 3.7 → 15.9 MB | +|---|---|---| +| libdeflate 1.19, whole buffer | 2.14–2.18x | 1.64–1.67x | +| ISA-L 2.30, whole buffer | 1.48–1.49x | 1.29–1.33x | +| ISA-L 2.30, streaming, 64 KB | 1.39–1.42x | 1.23–1.25x | +| `InflateStream` | 1.56–1.61x | 1.35–1.41x | + +`InflateStream` is a C# port of libdeflate's decoder design made resumable. It reaches roughly +three quarters of libdeflate's speed on XLSX and four fifths on XLSB, and passes ISA-L without a +native dependency. An earlier throwaway version that decoded a whole buffer in one call measured +1.74–1.78x on XLSX, so streaming costs under a tenth. + +End to end in the reader, in-process A/B over the real-data corpus, identical checksums. The +`InflateStream` column is one run of 30 rounds; the ISA-L column spans four runs: + +| read | `InflateStream` | ISA-L behind the same seam | +|---|---|---| +| XLSX, no prefetch | 1.25–1.26x | 1.19–1.21x | +| XLSB, no prefetch | 1.13–1.15x | 1.10–1.14x | +| XLSX, prefetch | 1.11–1.12x | 1.09–1.15x | +| XLSB, prefetch | 1.05–1.06x | 1.03–1.07x | + +This is the `I + P` model again. Without prefetch a faster `I` converts directly. With prefetch the +read is `max(I, P)`, both decoders land together, and what is left is the parse. + +`InflateBenchmark`, BenchmarkDotNet `--job Short`, in-process, same laptop: + +| entry | `DeflateStream` | `InflateStream` | ratio | +|---|---|---|---| +| XLSX sheet | 34.91 ms, 541 B | 21.71 ms, 332 B | 0.62 | +| XLSB sheet | 19.40 ms, 404 B | 13.68 ms, 270 B | 0.71 | +| string-heavy `sharedStrings.xml` | 16.10 ms, 280 B | 10.32 ms, 270 B | 0.64 | + +Measured and not kept: + +| attempt | result | +|---|---| +| `AggressiveOptimization` on the decode loop | no change, so the loop does not depend on dynamic PGO | +| `Vector128` match copy for offsets of 16 and up | no change on XLSX, slightly slower on XLSB | + +#### The decode loop's Tier1 disassembly (2026-10-09, Ryzen 7 5700X) + +`DOTNET_JitDisasm=FastLoop` on a scratch harness that inflates the XLSX sheet 150 times. The `(Tier1)` +listing is 955 bytes, and the loop **keeps every live value in a register**: `bitbuf`, `bitsleft`, both +cursors, the limits, the table base and the mask are all in registers, with eight callee-saved pushes in the +prologue and no stack traffic inside the loop. The spill problem the CSV scanner had is not here. Two +earlier listings are `Instrumented Tier0` and `Tier1-OSR`; the OSR one is the method's first long call, +so read the last listing. + +Where the time goes, from timestamps around the pieces, one XLSX sheet, 16.6 ms in total: + +| piece | ms | | +|---|---|---| +| `FastLoop` (1,042 calls) | 14.6 | 88% | +| 417 dynamic block headers and table builds | 0.7 | 4% | +| copy to the caller and slide the window (isolated) | 0.8 | 5% | + +So the streaming overhead the "Fourth round" blamed for part of the gap is ~5%, and the table builds +are not it either. The gap to libdeflate is inside the loop, at ~2.4 GB/s of output. + +What the listing does show is small. Each decoded entry pays a zero-extending `mov r12d, r15d` before +`sub rsi, r12`, because `bitsleft` is a `long` and the entry a `uint`. Making `bitsleft` a `uint`, as +libdeflate has it, removes that and measured 15.87–15.99 ms → 15.63–15.77 ms on XLSX (~1.5%) and nothing on XLSB. +Not applied. The literal path takes four jumps per run of literals because the JIT laid the second and +third literal out of line, and several `mov r13, rbx` / `shrx rbx, r13, r15` pairs keep the pre-shift +`saved` value alive for the match path whether or not it is used. None of that is something source +changes reach reliably; what is left is the JIT's block layout and register copies. Not worth chasing. + +**A behavior change came with it.** `DeflateStream` does not reject a stream that ends before its +final block: it returns what it decoded and reports end of stream. A sheet entry cut short therefore +used to yield a silent prefix of its rows. `InflateStream` throws `InvalidDataException`. + ## Known open items -- NativeAOT XLSX is ~13% behind the JIT, and the static profile does not close it (see "NativeAOT, - again"). Next step: diff the JIT's Tier1 code for `ParseRowInWindow` against ILC's output. -- 17-digit numbers are parsed twice in XLSX: emit rejects them to keep the text exact, and the - consumer parses them again. A `Cell` flag for an approximate double would save ~3 ms per - real-data read, at the cost of a wider `Cell`. +- **Parallel inflate is the open lead (spec B, not built).** It replaces the pipeline's producer only; the + queue, workers and merge stay. The oracle ceilings at 8 threads, parallel inflate plus parse, are 7.7 ms for + XLSX and 3.6 ms for XLSB, against 17.6 and 13.3 ms for the shipped pipeline. A deflate block finder costs + 0.4–1 ms per split point, with no false positives in 14 hits. The marker finding: XLSX back-references stop + depending on the unknown window after about 1 MB, but XLSB ones never do, so about 65–70% of XLSB output bytes depend on the unknown window. The cost of the 16-bit marker decode loop is unmeasured, and it decides whether + XLSB gains much. +- `InflateStream` reaches about three quarters of libdeflate's inflate speed on XLSX. The Tier1 + disassembly shows no spills and 88% of the time in `FastLoop` itself (see "The decode loop's Tier1 + disassembly"); the remainder is JIT codegen. A `uint bitsleft` is worth ~1.5% if anyone wants it. +- "Fourth round" is from the i7-1365U. The Ryzen 7 5700X re-run (2026-10-09, `--job Medium`) is in + `benchmarks.md`: `InflateStream` is 0.70x / 0.78x / 0.63x of `DeflateStream` on the XLSX sheet, XLSB + sheet and string-heavy `sharedStrings.xml`, against 0.62 / 0.71 / 0.64 on the laptop. The + non-prefetched reads moved with it (real-data XLSX 49.7 → 43.8 ms, XLSB 27.7 → 25.7 ms; string-heavy + XLSX 41.2 → 33.2 ms) and the prefetched ones did not (27.8 → 29.5 ms, 16.9 → 17.4 ms), as the + `max(I, P)` model predicts. The native C ABI table in `benchmarks.md` has not been re-run since. +- NativeAOT XLSX is ~7–10% behind the JIT, down from the ~13% in "NativeAOT, again"; the diff of the + JIT's Tier1 `ParseRowInWindow` against ILC's is done (see "NativeAOT XLSX gap, re-checked"). The one + difference found does not move the shipped library. +- **17-digit numbers parsed twice in XLSX: prototyped 2026-10-09 (Ryzen 7 5700X), not kept.** Emit + rejects them to keep the text exact and the consumer parses them again. The prototype parsed up to + 19 digits at emit, had `FastDouble` report whether the result keeps every digit (15 or fewer), and + carried that as a `NumberApproximate` flag through `CellAccumulator`, `CellDesc` and `Cell`. + `TryGetDouble` and `TryParse` used the number; every other `T` ignored an approximate one and + read the text, so `long` and `decimal` stayed exact. The flag costs nothing in size: `CellDesc` stays + 32 bytes (the new `bool` fills the last padding byte) and `Cell` has padding to spare, so "a wider + `Cell`" was wrong. Real-data XLSX, 49,799 17-digit cells of 589,815, min of 35 rounds, base and new + `ExcelReader.Core.dll` alternated three times in separate processes, identical checksums: + + | consumer | before | after | + |---|---|---| + | `TryParse` on every number | 43.3–43.6 ms | 42.1–43.1 ms (~2%) | + | `TryGetDouble` on every number | 42.4–43.0 ms | 42.2–42.4 ms (~1%) | + | `TryParse` | 53.5–54.5 ms | 54.3–54.8 ms (~+1%) | + | no number parsed | 41.5–41.9 ms | 42.1–43.0 ms (~+1.5%) | + + A trade, like lazy number parsing: the double consumers win ~1 ms, not the ~3 ms estimated, because + the 36 ns parse moves to emit and the consumers that never ask for a double (`decimal`, `long`, + text) now pay it for nothing. Not worth a flag threaded through three structs and a `Cell` + semantics rule. Reopen only if double columns dominate real workloads. - `PrefetchStream` handoff: ~2.7 ms on the i7, ≤2.1 ms on the Ryzen string-heavy sheet, of which the copy itself is ~0.3 ms. Removing it means replacing the `Stream` seam with a buffer-exchange protocol — and that seam is where the decompressed-byte limit counters sit, so it is a trust boundary, not just a copy. -- **Needs re-measuring.** This reading predates `CsvStructuralScanner` (de49a45, 2026-09-22), which - moved the enumerator off `CsvControlScanner` and made `Reset` per buffer rather than per record, - so the write barrier named below is likely already gone. The 1BRC re-run above found 1.36x on - rows-only enumeration over the same period, so the gap quoted here is stale. - After `DrainFields`, rows-only enumeration of wide rows still runs ~1.75x the framing prototype. - What remains is per record rather than per field (the `MoveNext` → `TryParseRecordFromBuffer` → - `TryParseSimpleRecord` call chain, `BeginRecord`, the write barrier from `CsvControlScanner.Continue` - storing the buffer reference every record) plus the consumer side: `Row` indexer access costs - about as much again as enumeration on the wide shape. +- **CSV enumerator overhead, re-measured 2026-10-09 (Ryzen 7 5700X).** The earlier reading is + superseded: it predates `CsvStructuralScanner` and its per-record diagnosis no longer holds. + Scratch harness, min of 50 interleaved rounds, 50,000 rows, the scanner driven directly for the + framing floor (drain the field-end bits, emit a descriptor per field): + + | | wide, 32 fields (9.8 MB) | narrow, 4 fields (1.2 MB) | + |---|---|---| + | scanner only, popcount | 0.79 ms | 0.10 ms | + | scan + drain + emit a 32-byte descriptor | 2.93 ms | 0.38 ms | + | reader, rows only | 5.37 ms (1.83x) | 0.88 ms (2.3x) | + | reader + `row[i].Value.Length` on every cell | 7.72 ms | 1.10 ms | + + The overhead above framing is **per field, not per record**: fitting the two shapes gives ~1.4 ns + per field and ~4 ns per row. Per field, `DrainFields` also loads `buf[stop]` to find the record end, + `buf[start]` to test for a quote, and builds a `CellDesc` with a branch on empty; the prototype does + none of that. The indexer adds ~1.4 ns per cell on top, about 45% of rows-only, not "as much again". + Hoisting `_delimiter` and `_quote` into locals made it **slower** (5.37 → 5.6–5.7 ms), the register + pressure result from the CSV enumerator section again. Telling quoted from unquoted fields from a + scanner quote-start mask instead of the `buf[start]` load is the one idea left; it is worth at most + ~0.5 ns of a ~3.3 ns field, which is a few percent of any read that converts values. - Reading a CSV from an already-resident buffer is slower than from a stream for spans only, and faster for full conversion. Reproduced in both directions across four runs, unexplained. diff --git a/docs/snippets/Csv.cs b/docs/snippets/Csv.cs new file mode 100644 index 00000000..73013dc5 --- /dev/null +++ b/docs/snippets/Csv.cs @@ -0,0 +1,158 @@ +namespace Snippets.Csv +{ + internal sealed class ChangeRow + { + [ExcelColumn("file")] + public string File { get; set; } = ""; + + [ExcelColumn("lines_added")] + public int LinesAdded { get; set; } + } +} + +namespace Snippets.Csv.Read +{ + #region csv-read + using ExcelReader.Core.Parser; + using ExcelReader.Core.Reader; + #endregion + + internal static class Snippet + { + internal static void Run() + { + #region csv-read + using var csv = Excel.FromCsvFile("report.csv"); + + foreach (var row in csv.FirstSheet) + { + Console.WriteLine(row[0].GetString()); + } + + // Typed parsing works exactly like the Excel workbooks: + foreach (var item in ExcelParser.FromAttributes().Parse(csv.FirstSheet)) + { + Console.WriteLine($"{item.File}: +{item.LinesAdded}"); + } + #endregion + } + } +} + +namespace Snippets.Csv.ReadOptions +{ + internal static class Snippet + { + internal static void Run() + { + #region csv-read-options + var options = new ExcelReaderOptions { Csv = new CsvReaderOptions { Delimiter = (byte)';' } }; + using var csv = Excel.FromCsvFile("relatorio.csv", options); + #endregion + } + } +} + +namespace Snippets.Csv.OpenByFormat +{ + internal static class Snippet + { + internal static void Run(string path, string secret) + { + #region csv-open-by-format + ExcelFileFormat format = Path.GetExtension(path) is ".csv" ? ExcelFileFormat.Csv : ExcelFileFormat.Unknown; + + using IExcelWorkbook workbook = Excel.Open(path, format, new ExcelReaderOptions + { + Password = secret, // used by the Excel formats + Csv = CsvReaderOptions.Default with { SniffDialect = true }, // used by the CSV format + }); + #endregion + } + } +} + +namespace Snippets.Csv.Sniff +{ + #region csv-sniff + using ExcelReader.Core.Reader; + using ExcelReader.Core.Reader.Csv; + #endregion + + internal static class Snippet + { + internal static void Run() + { + #region csv-sniff + CsvDialect dialect = CsvSniffer.DetectFile("export.csv"); + using var csv = Excel.FromCsvFile("export.csv", new ExcelReaderOptions { Csv = CsvReaderOptions.Default.WithDialect(dialect) }); + #endregion + } + } +} + +namespace Snippets.Csv.SniffOnOpen +{ + internal static class Snippet + { + internal static void Run() + { + #region csv-sniff-on-open + using var csv = Excel.FromCsvFile("export.csv", new ExcelReaderOptions { Csv = CsvReaderOptions.Default with { SniffDialect = true } }); + #endregion + } + } +} + +namespace Snippets.Csv.Write +{ + #region csv-write + using ExcelReader.Core.Writer; + using ExcelReader.Core.Writer.Csv; + #endregion + + internal static class Snippet + { + internal static void Run() + { + #region csv-write + using var stream = File.Create("out.csv"); + using var workbook = CsvWorkbookWriter.Create(stream); + using var writer = workbook.AddSheet("Sheet1"); + + using (CsvRowWriter row = writer.StartRow()) + { + row.Write("Name"); + row.Write("Total"); + row.Write("Created"); + } + + using (CsvRowWriter row = writer.StartRow()) + { + row.Write("Q1"); + row.Write(42); + row.Write(DateTime.UtcNow); + } + #endregion + } + } +} + +namespace Snippets.Csv.WriteOptions +{ + internal static class Snippet + { + internal static void Run(Stream stream) + { + #region csv-write-options + using var workbook = CsvWorkbookWriter.Create(stream, leaveOpen: false, new CsvWriterOptions + { + Delimiter = (byte)';', + Encoding = Encoding.Latin1, // fields are formatted as UTF-8, then transcoded on the way out + WriteByteOrderMark = true, // what makes Excel open a UTF-8 file as UTF-8 + NewLine = CsvNewLine.LineFeed, // default is CRLF, as RFC 4180 specifies + }); + #endregion + } + } +} diff --git a/docs/snippets/Encryption.cs b/docs/snippets/Encryption.cs new file mode 100644 index 00000000..8d7c2fb8 --- /dev/null +++ b/docs/snippets/Encryption.cs @@ -0,0 +1,78 @@ +namespace Snippets.Encryption.Open +{ + internal static class Snippet + { + internal static void Run() + { + #region encryption-open + var options = new ExcelReaderOptions { Password = "hunter2" }; + using IExcelWorkbook workbook = Excel.Open("protected.xlsx", options); + foreach (Row row in workbook.FirstSheet) { /* ... */ } + #endregion + } + } +} + +namespace Snippets.Encryption.Reason +{ + internal static class Snippet + { + internal static void Run(string path, ExcelReaderOptions options) + { + #region encryption-reason + try + { + using IExcelWorkbook workbook = Excel.Open(path, options); + } + catch (ExcelEncryptionException ex) when (ex.Reason is ExcelEncryptionReason.PasswordIncorrect) + { + // Ask again. UnsupportedScheme and IntegrityFailure are terminal. + } + #endregion + } + } +} + +namespace Snippets.Encryption.VerifyIntegrity +{ + internal static class Snippet + { + internal static void Run() + { + #region encryption-verify-integrity + var options = new ExcelReaderOptions + { + Password = "hunter2", + VerifyEncryptedIntegrity = true, // pay one full pass; reject tampered packages + }; + #endregion + } + } +} + +namespace Snippets.Encryption.Write +{ + #region encryption-write + using ExcelReader.Core.Reader; + using ExcelReader.Core.Writer; + using ExcelReader.Core.Writer.Xlsx; + #endregion + + internal static class Snippet + { + internal static async Task Run() + { + #region encryption-write + using var plain = new MemoryStream(); + await using (var workbook = XlsxWorkbookWriter.Create(plain, leaveOpen: true)) + { + // write sheets and rows as usual + } + plain.Position = 0; + + using var destination = File.Create("secret.xlsx"); + await Excel.EncryptPackageAsync(plain, destination, "hunter2"); + #endregion + } + } +} diff --git a/docs/snippets/ExcelReader.Snippets.csproj b/docs/snippets/ExcelReader.Snippets.csproj new file mode 100644 index 00000000..306075ec --- /dev/null +++ b/docs/snippets/ExcelReader.Snippets.csproj @@ -0,0 +1,20 @@ + + + + + net10.0 + false + + false + + + + + + + + + diff --git a/docs/snippets/GlobalUsings.cs b/docs/snippets/GlobalUsings.cs new file mode 100644 index 00000000..2b80db1b --- /dev/null +++ b/docs/snippets/GlobalUsings.cs @@ -0,0 +1,17 @@ +// Most guide blocks are fragments that leave their using directives out; the ones that show them +// still compile against these, so a missing directive in a block is not caught here. +global using ExcelReader.Arrow; +global using ExcelReader.Core; +global using ExcelReader.Core.Parser; +global using ExcelReader.Core.Reader; +global using ExcelReader.Core.Reader.Csv; +global using ExcelReader.Core.Reader.Schema; +global using ExcelReader.Core.Reader.Xls; +global using ExcelReader.Core.Reader.Xlsb; +global using ExcelReader.Core.Reader.Xlsx; +global using ExcelReader.Core.Writer; +global using ExcelReader.Core.Writer.Csv; +global using ExcelReader.Core.Writer.Xls; +global using ExcelReader.Core.Writer.Xlsb; +global using ExcelReader.Core.Writer.Xlsx; +global using System.Text; diff --git a/docs/snippets/Parsing.cs b/docs/snippets/Parsing.cs new file mode 100644 index 00000000..72dc9112 --- /dev/null +++ b/docs/snippets/Parsing.cs @@ -0,0 +1,291 @@ +namespace Snippets.Parsing +{ + internal sealed class ChangeRow + { + [ExcelColumn("file")] + public string File { get; set; } = ""; + + [ExcelColumn("lines_added")] + public int LinesAdded { get; set; } + } + + internal readonly ref struct ChangeRowRef + { + public ReadOnlySpan File { get; init; } + public int LinesAdded { get; init; } + } +} + +namespace Snippets.Parsing.Attributes +{ + #region parsing-attributes + using ExcelReader.Core.Parser; + using ExcelReader.Core.Reader; + #endregion + + #region parsing-attributes + public sealed class ChangeRow + { + [ExcelColumn("file")] + public string File { get; set; } = ""; + + [ExcelColumn("lines_added")] + public int LinesAdded { get; set; } + } + #endregion + + internal static class Snippet + { + internal static void Run() + { + #region parsing-attributes + using var workbook = Excel.FromXlsxFile("changes.xlsx"); + var parser = ExcelParser.FromAttributes(); + + foreach (var item in parser.Parse(workbook.FirstSheet)) + { + Console.WriteLine($"{item.File}: +{item.LinesAdded}"); + } + #endregion + } + } +} + +namespace Snippets.Parsing.AttributesOpen +{ + internal static class Snippet + { + internal static void Run() + { + #region parsing-attributes-open + using IExcelWorkbook workbook = Excel.Open("changes.xlsx"); // or .xlsb / .xls + foreach (var item in ExcelParser.FromAttributes().Parse(workbook.FirstSheet)) { /* ... */ } + #endregion + } + } +} + +namespace Snippets.Parsing.Generated +{ + #region parsing-generated + using ExcelReader.Core.Parser; + using ExcelReader.Core.Reader; + using ExcelReader.Core.Writer; + using ExcelReader.Core.Writer.Xlsx; + #endregion + + #region parsing-generated + [ExcelSerializable] // the model must be declared partial + public partial class ChangeRow + { + [ExcelColumn("file")] + public string File { get; set; } = ""; + + [ExcelColumn("lines_added")] + public int LinesAdded { get; set; } + } + #endregion + + internal static class Snippet + { + internal static async Task Run() + { + #region parsing-generated + using var workbook = Excel.FromXlsxFile("changes.xlsx"); + foreach (var item in ExcelParser.Generated().Parse(workbook.FirstSheet)) + { + Console.WriteLine($"{item.File}: +{item.LinesAdded}"); + } + + var changes = new[] { new ChangeRow { File = "README.md", LinesAdded = 12 } }; + await using var stream = File.Create("changes.xlsx"); + await using var writer = XlsxWorkbookWriter.Create(stream); // or XlsbWorkbookWriter / XlsWorkbookWriter / CsvWorkbookWriter + await using (var sheet = writer.AddSheet("Changes")) + { + await sheet.WriteRecordsAsync(changes, ExcelRecordLayout.Generated()); + } + #endregion + } + } +} + +namespace Snippets.Parsing.Fluent +{ + #region parsing-fluent + using ExcelReader.Core.Parser; + using ExcelReader.Core.Reader; + #endregion + + #region parsing-fluent + public sealed class ChangeRow + { + public string File { get; set; } = ""; + public int LinesAdded { get; set; } + } + #endregion + + internal static class Snippet + { + internal static void Run() + { + #region parsing-fluent + var parser = ExcelParser.Build(builder => builder + .Factory(() => new ChangeRow()) + .Property(["file"], ExcelCellReaders.String, (ref ChangeRow r, string v) => r.File = v) + .Property(["lines_added"], ExcelCellReaders.Parsable, (ref ChangeRow r, int v) => r.LinesAdded = v)); + + using var workbook = Excel.FromXlsxFile("changes.xlsx"); + foreach (var item in parser.Parse(workbook.FirstSheet)) + { + Console.WriteLine($"{item.File}: +{item.LinesAdded}"); + } + #endregion + } + } +} + +namespace Snippets.Parsing.FluentIndex +{ + internal static class Snippet + { + internal static void Run() + { + #region parsing-fluent-index + var parser = ExcelParser.Build(builder => builder + .Factory(() => new ChangeRow()) + .PropertyAt(0, ExcelCellReaders.String, (ref ChangeRow r, string v) => r.File = v) + .PropertyAt(1, ExcelCellReaders.Parsable, (ref ChangeRow r, int v) => r.LinesAdded = v)); + #endregion + } + } +} + +namespace Snippets.Parsing.FluentFallback +{ + internal static class Snippet + { + internal static void Run() + { + #region parsing-fluent-fallback + var parser = ExcelParser.BuildWithAttributeFallback(builder => builder + .Property(["file"], ExcelCellReaders.String, (ref ChangeRow r, string v) => r.File = v.ToUpperInvariant())); + #endregion + } + } +} + +namespace Snippets.Parsing.Options +{ + #region parsing-options + using System.Globalization; + using ExcelReader.Core.Parser; + #endregion + + internal static class Snippet + { + internal static void Run() + { + #region parsing-options + var options = new ExcelParserOptions + { + HeaderRow = 1, // 1-based row holding the headers + ColumnNameComparer = StringComparer.OrdinalIgnoreCase, + HeaderNormalization = HeaderNormalization.Trim | HeaderNormalization.CollapseSpaces, + Culture = CultureInfo.GetCultureInfo("pt-BR"), // parse "1.234,56" as 1234.56m + }; + + var parser = ExcelParser.FromAttributes(options); + #endregion + } + } +} + +namespace Snippets.Parsing.Required +{ + #region parsing-required + public sealed class Order + { + [ExcelRequired] + public int Id { get; set; } + + [ExcelRequired(AllowEmpty = true)] // column must exist; blank cells allowed + public string? Note { get; set; } + } + #endregion +} + +namespace Snippets.Parsing.Converter +{ + #region parsing-converter + using System.Globalization; + using ExcelReader.Core.Parser; + using ExcelReader.Core.Reader; + #endregion + + #region parsing-converter + public sealed class BrlMoneyConverter : IExcelCellConverter + { + public bool TryConvert(in Cell cell, in ExcelRowContext context, out decimal value) + { + string text = cell.GetString().Replace("R$", "", StringComparison.Ordinal).Trim(); + return decimal.TryParse(text, NumberStyles.Currency, CultureInfo.GetCultureInfo("pt-BR"), out value); + } + } + + public sealed class Invoice + { + [ExcelConverter(typeof(BrlMoneyConverter))] + public decimal Total { get; set; } + } + #endregion +} + +namespace Snippets.Parsing.RefStruct +{ + #region parsing-ref-struct + using System.Text; + using ExcelReader.Core.Parser; + using ExcelReader.Core.Reader; + #endregion + + #region parsing-ref-struct + public readonly ref struct ChangeRowRef + { + public ReadOnlySpan File { get; init; } // zero-copy — aliases the reader's row buffer + public int LinesAdded { get; init; } + } + #endregion + + internal static class Snippet + { + internal static void Run() + { + #region parsing-ref-struct + using var workbook = Excel.FromXlsxFile("changes.xlsx"); + + foreach (ChangeRowRef item in ExcelParser.FromAttributes().Parse(workbook.FirstSheet)) + { + Console.WriteLine($"{Encoding.UTF8.GetString(item.File)}: +{item.LinesAdded}"); + } + #endregion + } + } +} + +namespace Snippets.Parsing.RefStructAsync +{ + internal static class Snippet + { + internal static async Task Run() + { + #region parsing-ref-struct-async + await using var workbook = await Excel.FromXlsxFileAsync("changes.xlsx"); + + await foreach (ChangeRowRef item in ExcelParser.FromAttributes().Parse(workbook.FirstSheet)) + { + Console.WriteLine($"{Encoding.UTF8.GetString(item.File)}: +{item.LinesAdded}"); + } + #endregion + } + } +} diff --git a/docs/snippets/Reading.cs b/docs/snippets/Reading.cs new file mode 100644 index 00000000..fd91e128 --- /dev/null +++ b/docs/snippets/Reading.cs @@ -0,0 +1,384 @@ +namespace Snippets.Reading.ReadRows +{ + #region reading-read-rows + using ExcelReader.Core.Reader; + #endregion + + internal static class Snippet + { + internal static void Run() + { + #region reading-read-rows + using var workbook = Excel.FromXlsxFile("report.xlsx"); + + foreach (var row in workbook.FirstSheet) + { + string name = row[0].GetString(); + + if (row[1].TryParse(null, out int quantity)) + { + Console.WriteLine($"{name}: {quantity}"); + } + + if (row[2].Type == CellType.Date && row[2].TryGetDateTime(workbook.IsDate1904, out var date)) + { + Console.WriteLine(date); + } + } + #endregion + } + } +} + +namespace Snippets.Reading.ParallelParse +{ + internal sealed class SalesRow + { + public decimal Revenue { get; set; } + } + + internal static class Snippet + { + private static decimal Total; + + internal static async Task Run() + { + #region reading-parallel-parse + using XlsxWorkbook workbook = Excel.FromXlsxFile("big.xlsx"); + + await foreach (var row in SheetParallel.ParseAsync(workbook.FirstSheet, ExcelParser.FromAttributes(), new SheetParallelOptions { DegreeOfParallelism = 8 })) + { + Total += row.Revenue; + } + #endregion + } + } +} + +namespace Snippets.Reading.ParallelAggregate +{ + #region reading-parallel-aggregate + public ref struct SaleRow + { + [ExcelColumn("Region")] public ReadOnlySpan Region { get; set; } + [ExcelColumn("Units")] public int Units { get; set; } + } + + public sealed class UnitTotal : IRowAccumulator + { + public long Units; + + public void Add(SaleRow model) => Units += model.Units; + + public void Merge(UnitTotal following) => Units += following.Units; + } + #endregion + + internal static class Snippet + { + internal static async Task Run() + { + #region reading-parallel-aggregate + using XlsxWorkbook workbook = Excel.FromXlsxFile("big.xlsx"); + + UnitTotal total = await SheetParallel.AggregateAsync( + workbook.FirstSheet, + ExcelParser.FromAttributes(), + new SheetParallelOptions { HeaderRow = 1 }); + #endregion + } + } +} + +namespace Snippets.Reading.ParallelForeach +{ + using Snippets.Reading.ParallelAggregate; + + internal static class Snippet + { + internal static async Task Run() + { + #region reading-parallel-foreach + using XlsxWorkbook workbook = Excel.FromXlsxFile("big.xlsx"); + + long total = 0; + await SheetParallel.ForEachAsync( + workbook.FirstSheet, + ExcelParser.FromAttributes(), + row => Interlocked.Add(ref total, row.Units), + new SheetParallelOptions { DegreeOfParallelism = 8, HeaderRow = 1 }); + #endregion + } + } +} + +namespace Snippets.Reading.ArrowRead +{ + #region reading-arrow-read + using ExcelReader.Arrow; + using ExcelReader.Core.Reader; + #endregion + + internal static class Snippet + { + internal static void Run() + { + #region reading-arrow-read + using var workbook = Excel.FromXlsxFile("report.xlsx"); + Apache.Arrow.RecordBatch batch = workbook.FirstSheet.ToArrowRecordBatch(); + #endregion + } + } +} + +namespace Snippets.Reading.ArrowWrite +{ + #region reading-arrow-write + using ExcelReader.Arrow; + using ExcelReader.Core.Writer; + using ExcelReader.Core.Writer.Xlsx; + #endregion + + internal static class Snippet + { + internal static void Run(Apache.Arrow.RecordBatch batch) + { + #region reading-arrow-write + using XlsxWorkbookWriter workbook = XlsxWorkbookWriter.Create(File.Create("report.xlsx")); + workbook.WriteRecordBatch(batch); // adds the sheet, writes the header + every row, ends the workbook + #endregion + } + } +} + +namespace Snippets.Reading.Open +{ + #region reading-open + using ExcelReader.Core.Reader; + #endregion + + internal static class Snippet + { + internal static void Run() + { + #region reading-open + using IExcelWorkbook workbook = Excel.Open("report.xlsx"); // or report.xlsb / report.xls + + foreach (var row in workbook.FirstSheet) + { + Console.WriteLine(row[0].GetString()); + } + #endregion + } + } +} + +namespace Snippets.Reading.OpenEverySheet +{ + internal static class Snippet + { + internal static void Run() + { + #region reading-open-every-sheet + using IExcelWorkbook workbook = Excel.Open("report.xlsx"); + + for (int i = 0; i < workbook.SheetCount; i++) + { + IExcelSheet sheet = workbook.SheetAt(i); + Console.WriteLine(sheet.Name); + foreach (var row in sheet) + { + Console.WriteLine(row[0].GetString()); + } + } + #endregion + } + } +} + +namespace Snippets.Reading.OpenTryGetSheet +{ + internal static class Snippet + { + internal static void Run(IExcelWorkbook workbook) + { + #region reading-open-try-get-sheet + if (workbook.TryGetSheet("Totals", out var totals)) + { + foreach (var row in totals) + { + Console.WriteLine(row[0].GetString()); + } + } + #endregion + } + } +} + +namespace Snippets.Reading.OpenVisibility +{ + #region reading-open-visibility + using ExcelReader.Core; // ExcelSheetVisibility, shared by the readers and writers + #endregion + + internal static class Snippet + { + internal static void Run(IExcelWorkbook workbook) + { + #region reading-open-visibility + for (int i = 0; i < workbook.SheetCount; i++) + { + IExcelSheet sheet = workbook.SheetAt(i); + if (sheet.Visibility != ExcelSheetVisibility.Visible) + { + continue; // skip hidden and veryHidden sheets + } + Console.WriteLine(sheet.Name); + } + #endregion + } + } +} + +namespace Snippets.Reading.OpenCsv +{ + internal static class Snippet + { + internal static void Run() + { + #region reading-open-csv + using IExcelWorkbook workbook = Excel.Open("report.csv", ExcelFileFormat.Csv); + #endregion + } + } +} + +namespace Snippets.Reading.Table +{ + internal sealed class ProductRow + { + public string Product { get; set; } = ""; + public int Qty { get; set; } + } + + internal static class Snippet + { + internal static void Run() + { + #region reading-table + using IExcelWorkbook workbook = Excel.Open("sales.xlsx"); + if (workbook.TryGetTable("Sales", out ExcelTable? sales)) + { + foreach (ProductRow row in ExcelParser.FromAttributes().Parse(sales.AsSheet())) + { + Console.WriteLine($"{row.Product}: {row.Qty}"); + } + } + #endregion + } + } +} + +namespace Snippets.Reading.SheetsInParallel +{ + internal static class Snippet + { + internal static void Run(string path) + { + #region reading-sheets-in-parallel + using XlsxWorkbook workbook = Excel.FromXlsxFile(path); + + Parallel.For(0, workbook.Sheets.Count, i => + { + foreach (Row row in workbook.Sheets[i]) + { + // consume + } + }); + #endregion + } + } +} + +namespace Snippets.Reading.Async +{ + #region reading-async + using ExcelReader.Core.Reader; + #endregion + + internal static class Snippet + { + internal static async Task Run(CancellationToken cancellationToken) + { + #region reading-async + await using var workbook = await Excel.FromXlsxFileAsync("report.xlsx", ct: cancellationToken); + + await foreach (var row in workbook.FirstSheet) + { + Console.WriteLine(row[0].GetString()); + } + #endregion + } + } +} + +namespace Snippets.Reading.AsyncEnumerator +{ + internal static class Snippet + { + internal static async Task Run(CancellationToken cancellationToken) + { + #region reading-async-enumerator + await using var workbook = await Excel.FromXlsxFileAsync("report.xlsx", ct: cancellationToken); + await using var rows = workbook.FirstSheet.GetAsyncEnumerator(cancellationToken); + + while (await rows.MoveNextAsync()) + { + var row = rows.Current; + Console.WriteLine(row[0].GetString()); + } + #endregion + } + } +} + +namespace Snippets.Reading.Prefetch +{ + internal static class Snippet + { + internal static void Run() + { + #region reading-prefetch + var options = new ExcelReaderOptions { PrefetchDecompression = true }; + using var workbook = Excel.FromXlsxFile("report.xlsx", options); + + foreach (var row in workbook.FirstSheet) + { + Console.WriteLine(row[0].GetString()); + } + #endregion + } + } +} + +namespace Snippets.Reading.DataReader +{ + #region reading-data-reader + using System.Data; + using ExcelReader.Core.Reader; + #endregion + + internal static class Snippet + { + internal static void Run() + { + #region reading-data-reader + using IExcelWorkbook workbook = Excel.Open("report.xlsx"); + using IDataReader data = new SheetDataReader(workbook.FirstSheet); // header on row 1 by default + + var table = new DataTable(); + table.Load(data); + #endregion + } + } +} diff --git a/docs/snippets/UntrustedInput.cs b/docs/snippets/UntrustedInput.cs new file mode 100644 index 00000000..c894b537 --- /dev/null +++ b/docs/snippets/UntrustedInput.cs @@ -0,0 +1,69 @@ +namespace Snippets.UntrustedInput.Options +{ + internal static class Snippet + { + internal static ExcelReaderOptions Run() + { + #region untrusted-input-options + var uploadOptions = new ExcelReaderOptions + { + MaxTotalDecompressedBytes = 64L * 1024 * 1024, // everything one workbook may inflate + MaxSharedStringBytes = 16L * 1024 * 1024, + MaxCellBytes = 1024 * 1024, + MaxZipEntries = 1_000, + }; + #endregion + return uploadOptions; + } + } +} + +namespace Snippets.UntrustedInput.Reject +{ + internal static class Snippet + { + internal static void Run(Stream upload, ExcelReaderOptions uploadOptions) + { + #region untrusted-input-reject + try + { + using IExcelWorkbook workbook = Excel.Open(upload, options: uploadOptions); + foreach (Row row in workbook.FirstSheet) + { + // stage the row; commit only once the loop has finished + } + } + catch (ExcelLimitExceededException ex) + { + Console.WriteLine($"Rejected: {ex.LimitName} reached {ex.Actual}, limit {ex.Limit}"); + } + catch (Exception ex) when (IsRejectedFile(ex)) + { + Console.WriteLine($"Rejected: {ex.Message}"); + } + + static bool IsRejectedFile(Exception ex) => + ex is InvalidDataException or EndOfStreamException or NotSupportedException + or ExcelEncryptionException or ExcelParseException + || ex.GetType() == typeof(ArgumentException) + || ex.GetType() == typeof(InvalidOperationException); + #endregion + } + } +} + +namespace Snippets.UntrustedInput.CsvInjection +{ + internal static class Snippet + { + internal static void Run(CsvRowWriter row, string comment) + { + #region untrusted-input-csv-injection + static string Neutralize(string text) => + text.Length > 0 && "=+-@\t\r".Contains(text[0]) ? "'" + text : text; + + row.Write(Neutralize(comment)); + #endregion + } + } +} diff --git a/docs/snippets/Writing.cs b/docs/snippets/Writing.cs new file mode 100644 index 00000000..c69772fe --- /dev/null +++ b/docs/snippets/Writing.cs @@ -0,0 +1,280 @@ +namespace Snippets.Writing.Xlsx +{ + #region writing-xlsx + using ExcelReader.Core.Writer; + using ExcelReader.Core.Writer.Xlsx; + #endregion + + internal static class Snippet + { + internal static async Task Run() + { + #region writing-xlsx + await using var stream = File.Create("out.xlsx"); + await using var workbook = XlsxWorkbookWriter.Create(stream); + + await using (var sheet = workbook.AddSheet("Summary")) + { + await using (var row = await sheet.StartRowAsync()) + { + row.Write("Name"); + row.Write("Total"); + row.Write("Created"); + } + + await using (var row = await sheet.StartRowAsync()) + { + row.Write("Q1"); + row.Write(42); + row.Write(DateTime.UtcNow); + } + } + + await workbook.EndAsync(); + #endregion + } + } +} + +namespace Snippets.Writing.XlsxSharedStrings +{ + internal static class Snippet + { + internal static async Task Run(Stream stream) + { + #region writing-xlsx-shared-strings + await using var workbook = XlsxWorkbookWriter.Create(stream, options: new XlsxWriterOptions { UseSharedStrings = true }); + #endregion + } + } +} + +namespace Snippets.Writing.HiddenSheets +{ + #region writing-hidden-sheets + using ExcelReader.Core; // ExcelSheetVisibility, shared by the readers and writers + #endregion + + internal static class Snippet + { + internal static async Task Run(XlsxWorkbookWriter workbook) + { + #region writing-hidden-sheets + await using (var data = workbook.AddSheet("Data")) { /* rows */ } // visible + await using (var lookups = workbook.AddSheet("Lookups", ExcelSheetVisibility.Hidden)) { /* rows */ } + await using (var audit = workbook.AddSheet("Audit", ExcelSheetVisibility.VeryHidden)) { /* rows */ } // not in Excel's unhide dialog + #endregion + } + } +} + +namespace Snippets.Writing.Styles +{ + #region writing-styles + using ExcelReader.Core.Writer; + using ExcelReader.Core.Writer.Xlsx; + #endregion + + internal static class Snippet + { + internal static async Task Run(Stream stream) + { + #region writing-styles + await using var workbook = XlsxWorkbookWriter.Create(stream); + + int currency = workbook.AddStyle(new CellStyle { NumberFormat = "R$ #,##0.00" }); + int header = workbook.AddStyle(new CellStyle { Bold = true }); + + await using var sheet = workbook.AddSheet("Summary"); + sheet.SetColumnStyle(columnIndex: 1, currency); // before the first row + sheet.SetColumnWidth(columnIndex: 0, width: 20); + + await using (var row = await sheet.StartRowAsync(header)) + { + row.Write("Product"); + row.Write("Total"); + } + await using (var row = await sheet.StartRowAsync()) + { + row.Write("Widget"); + row.Write(1234.5); + } + + await workbook.EndAsync(); + #endregion + } + } +} + +namespace Snippets.Writing.Xlsb +{ + #region writing-xlsb + using ExcelReader.Core.Writer; + using ExcelReader.Core.Writer.Xlsb; + #endregion + + internal static class Snippet + { + internal static async Task Run() + { + #region writing-xlsb + await using var stream = File.Create("out.xlsb"); + await using var workbook = XlsbWorkbookWriter.Create(stream); + + await using (XlsbSheetWriter sheet = workbook.AddSheet("Summary")) + { + await using (XlsbRowWriter row = await sheet.StartRowAsync()) + { + row.Write("Name"); + row.Write("Total"); + row.Write("Created"); + } + } + + await workbook.EndAsync(); + #endregion + } + } +} + +namespace Snippets.Writing.Xls +{ + #region writing-xls + using ExcelReader.Core.Writer; + using ExcelReader.Core.Writer.Xls; + #endregion + + internal static class Snippet + { + internal static async Task Run() + { + #region writing-xls + await using var stream = File.Create("out.xls"); + await using var workbook = XlsWorkbookWriter.Create(stream); + + using (var sheet = workbook.AddSheet("Summary")) + { + using (var row = sheet.StartRow()) + { + row.Write("Name"); + row.Write("Total"); + row.Write("Created"); + } + + using (var row = sheet.StartRow()) + { + row.Write("Q1"); + row.Write(42); + row.Write(DateTime.UtcNow); + } + } + + await workbook.EndAsync(); + #endregion + } + } +} + +namespace Snippets.Writing.Records +{ + #region writing-records + using ExcelReader.Core.Writer; + using ExcelReader.Core.Writer.Xlsx; + #endregion + + #region writing-records + public sealed class Sale + { + public string? Region { get; set; } + public int Units { get; set; } + public decimal Revenue { get; set; } + public DateOnly Date { get; set; } + } + #endregion + + internal static class Snippet + { + internal static async Task Run() + { + #region writing-records + var sales = new[] + { + new Sale { Region = "North", Units = 42, Revenue = 1234.50m, Date = new DateOnly(2026, 1, 2) }, + new Sale { Region = "South", Units = 17, Revenue = 512.00m, Date = new DateOnly(2026, 1, 3) }, + }; + + await using var stream = File.Create("sales.xlsx"); + await using var workbook = XlsxWorkbookWriter.Create(stream); // or XlsbWorkbookWriter / XlsWorkbookWriter / CsvWorkbookWriter + await using (var sheet = workbook.AddSheet("Sales")) + { + await sheet.WriteRecordsAsync(sales, ExcelRecordLayout.FromAttributes()); + } + #endregion + } + } +} + +namespace Snippets.Writing.Table +{ + using Snippets.Writing.Records; + + internal static class Snippet + { + internal static void Run(Sale[] sales) + { + #region writing-table + using XlsxWorkbookWriter workbook = XlsxWorkbookWriter.Create(File.Create("report.xlsx")); + XlsxSheetWriter sheet = workbook.AddSheet("Report"); + sheet.BeginTable("Sales", ["Region", "Units", "Revenue"]); + foreach (Sale s in sales) + { + using XlsxRowWriter row = sheet.StartRow(); + row.Write(s.Region); + row.Write(s.Units); + row.Write(s.Revenue); + } + sheet.EndTable(); + #endregion + } + } +} + +namespace Snippets.Writing.TableRecords +{ + using Snippets.Writing.Records; + + internal static class Snippet + { + internal static async Task Run(XlsxSheetWriter sheet, Sale[] sales) + { + #region writing-table-records + await sheet.WriteTableAsync(sales, "Sales", ExcelRecordLayout.FromAttributes()); + #endregion + } + } +} + +namespace Snippets.Writing.Prefetch +{ + internal sealed record Metric(string Name, double Value); + + internal static class Snippet + { + internal static async Task Run(Stream stream, Metric[] records) + { + #region writing-prefetch + await using var wb = XlsxWorkbookWriter.Create(stream, leaveOpen: true, options: new XlsxWriterOptions { PrefetchWrite = true }); + XlsxSheetWriter sheet = wb.AddSheet("S1"); + + foreach (var record in records) + { + using XlsxRowWriter row = sheet.StartRow(); + row.Write(record.Name); + row.Write(record.Value); + } + + await sheet.EndAsync(); + #endregion + } + } +} diff --git a/docs/snippets/sync.py b/docs/snippets/sync.py new file mode 100644 index 00000000..e333a3b8 --- /dev/null +++ b/docs/snippets/sync.py @@ -0,0 +1,64 @@ +"""Copies the #regions of docs/snippets/*.cs into the guide code blocks that name them. + +A block opts in with `` on the line before its fence. Every region of that +name, in file order, is dedented and joined with a blank line, so a snippet can show its using +directives and type declarations above statements that have to live inside a method here. + + python docs/snippets/sync.py rewrite the guides + python docs/snippets/sync.py --check fail if a guide differs from its snippets +""" + +import pathlib +import re +import sys +import textwrap + +ROOT = pathlib.Path(__file__).resolve().parent +GUIDES = ROOT.parent / "guide" +BLOCK = re.compile(r"^\n```csharp\n.*?^```$", re.MULTILINE | re.DOTALL) + + +def regions(): + found = {} + for path in sorted(ROOT.glob("*.cs")): + name, lines = None, [] + for line in path.read_text(encoding="utf-8").splitlines(): + directive = line.strip() + if directive.startswith("#region "): + name, lines = directive.removeprefix("#region ").strip(), [] + elif directive.startswith("#endregion") and name: + found.setdefault(name, []).append(textwrap.dedent("\n".join(lines)).strip("\n")) + name = None + elif name: + lines.append(line) + return {name: "\n\n".join(parts) for name, parts in found.items()} + + +def main(): + check = "--check" in sys.argv[1:] + snippets = regions() + used, stale = set(), [] + for guide in sorted(GUIDES.glob("*.md")): + text = guide.read_text(encoding="utf-8") + + def fill(match): + name = match.group(1) + if name not in snippets: + sys.exit(f"{guide.name}: no #region {name} in docs/snippets") + used.add(name) + return f"\n```csharp\n{snippets[name]}\n```" + + updated = BLOCK.sub(fill, text) + if updated != text: + stale.append(guide.name) + if not check: + guide.write_text(updated, encoding="utf-8", newline="\n") + + unused = sorted(snippets.keys() - used) + if unused: + sys.exit("regions no guide shows: " + ", ".join(unused)) + if check and stale: + sys.exit("out of date with docs/snippets (run python docs/snippets/sync.py): " + ", ".join(stale)) + + +main() diff --git a/docs/template/public/main.css b/docs/template/public/main.css new file mode 100644 index 00000000..24c4b26c --- /dev/null +++ b/docs/template/public/main.css @@ -0,0 +1,9 @@ +#logo { + height: 32px; + width: auto; +} + +/* The binding READMEs open with the logo for their GitHub/PyPI/crates.io pages; the navbar already shows it. */ +article img[alt="ExcelReader logo"] { + display: none; +} diff --git a/docs/template/public/main.js b/docs/template/public/main.js new file mode 100644 index 00000000..d1f73a5a --- /dev/null +++ b/docs/template/public/main.js @@ -0,0 +1,13 @@ +// The Rust README doubles as the crate's doctests: rustdoc hides lines starting with "# " and reads +// fence attributes like "rust,no_run". Do the same here before highlighting runs. +export default { + start: () => { + for (const code of document.querySelectorAll('code[class^="lang-rust"]')) { + code.className = 'lang-rust'; + code.textContent = code.textContent + .split('\n') + .filter(line => !/^\s*#(\s|$)/.test(line)) + .join('\n'); + } + } +}; diff --git a/docs/toc.yml b/docs/toc.yml new file mode 100644 index 00000000..d3707a7d --- /dev/null +++ b/docs/toc.yml @@ -0,0 +1,48 @@ +- name: Guide + items: + - name: Reading rows + href: guide/reading.md + - name: Typed parsing + href: guide/parsing.md + - name: Writing workbooks + href: guide/writing.md + - name: CSV + href: guide/csv.md + - name: Encrypted workbooks + href: guide/encryption.md + - name: Errors across languages + href: guide/errors.md + - name: Reading untrusted files + href: guide/untrusted-input.md +- name: API + href: api/ +- name: Bindings + items: + - name: Python + href: ../python/README.md + - name: Python API + href: python-api/ + - name: Rust + href: ../rust/excelreader/README.md + - name: Rust API (docs.rs) + href: https://docs.rs/excelreader + - name: C++ + href: ../cpp/README.md + - name: C++ API + href: cpp-api/ +- name: Performance + items: + - name: Benchmarks + href: performance/benchmarks.md + - name: Reader performance + href: performance/reader-performance.md + - name: Live results + href: https://gabrielmarquezmatte.github.io/ExcelReader/dev/bench/ +- name: Releases + items: + - name: Release notes + href: https://github.com/GabrielMarquezMatte/ExcelReader/releases + - name: Migrating to v7 + href: guide/migrating-to-v7.md + - name: Migrating to v6 + href: guide/migrating-to-v6.md diff --git a/python/README.md b/python/README.md index 74160f1e..16bf8f9f 100644 --- a/python/README.md +++ b/python/README.md @@ -1,3 +1,5 @@ +![ExcelReader logo](https://raw.githubusercontent.com/GabrielMarquezMatte/ExcelReader/master/assets/icon.png) + # excelreader (Python) Read and write XLSX, XLSB, XLS and CSV through ExcelReader's NativeAOT library. No .NET runtime @@ -175,11 +177,11 @@ once, typed or Arrow (see [Arrow](#arrow) below), and none disturbs another. Being a generator, `iter_parse_typed()` opens nothing until the first iteration, so a bad `batch_size` is only raised there, not at the call. -#### Reading a large CSV on several threads +#### Reading a large sheet on several threads `parse_typed()`, `to_arrow()`, `to_record_batch()` and `to_polars()` take `parallelism`: `1` (the -default) reads on one thread, `0` uses every core, `n` up to n threads. Only CSV is split. Any other -format, or a CSV under 256 KiB, is read on one thread with the same result. The table is identical +default) reads on one thread, `0` uses every core, `n` up to n threads. XLSX, XLSB and CSV are split. XLS, or a +sheet under 256 KiB, is read on one thread with the same result. The table is identical either way. Every partition's columns are held until they are merged, so the peak memory is higher than a sequential read of the same file. `to_polars()` with `parallelism` other than 1 parses the whole file instead of streaming it in batches. @@ -191,6 +193,11 @@ file instead of streaming it in batches. | `65K_Records_Data.csv` (8 MB, 65,535 rows) | 21.4 ms | 5.8 ms | 3.7x | | the same rows ×20 (160 MB, 1.3M rows) | 425.9 ms | 117.5 ms | 3.6x | +The same call on the real-data workbooks (XLSX 6.7 MB compressed, XLSB 3.7 MB; measured with the +published NativeAOT library): XLSX 54.1 ms at +`parallelism=1` against 20.3 ms at `0` (2.6x), XLSB 28.2 ms against 15.2 ms (1.85x). Inflate runs on +one thread, so under the JIT the sheet stage stops scaling at about three threads. + Peak working set on the 160 MB file, above the interpreter's own, went from 470 MiB to 503 MiB (both include the file's bytes and the handle's copy of them). @@ -380,6 +387,49 @@ with open_bytes(payload) as workbook: ... ``` +### From your own bytes + +`open_source()` takes any object with a `size` and a `read_at(offset, buffer)` that fills `buffer` +and returns the count. The library asks for 4 MiB blocks and caches them, and every sheet can still +be read on its own thread. + +```python +import urllib.request + +from excelreader import open_source + +class HttpSource: + def __init__(self, url): + self.url = url + with urllib.request.urlopen(urllib.request.Request(url, method="HEAD")) as response: + self.size = int(response.headers["Content-Length"]) + + def read_at(self, offset, buffer): + request = urllib.request.Request(self.url, headers={"Range": f"bytes={offset}-{offset + len(buffer) - 1}"}) + with urllib.request.urlopen(request) as response: + data = response.read() + buffer[: len(data)] = data + return len(data) + +with open_source(HttpSource("https://example.com/book.xlsx")) as workbook: + ... +``` + +`open_stream()` reads anything with `readinto()` or `read()`, such as `sys.stdin.buffer`. A CSV is +read as it arrives, and its first 16 MiB are kept so that `infer_schema()` followed by a read works; +an XLSX, XLSB or XLS is read whole first. + +```python +import sys + +from excelreader import open_stream + +with open_stream(sys.stdin.buffer, "csv") as workbook: + ... +``` + +Both callbacks run with the GIL held. For a file on disk, `open_workbook()` is faster. + ### Reader options `open_workbook()`/`open_bytes()` take an optional `OpenOptions` for CSV dialect settings and reader @@ -465,57 +515,74 @@ secret = encrypt_package_bytes(write_workbook_to_bytes(table, types, format="xls `benchmarks/bench_read.py` and `benchmarks/bench_write.py` over `tests/ExcelReader.Benchmarks/Data/65K_Records_Data.xlsb` (65,535 data rows, 14 columns), the same -fixture the .NET, C++ and Rust suites use. Measured on Windows 10 (22H2), 16 logical CPUs -@ 3.39 GHz, CPython 3.14.4, 10 runs each (medians shown; `min` is in the scripts' own output). -Competitors: polars 1.44.1 (with fastexcel 0.21.0), pandas 3.0.5 (with openpyxl 3.1.5), xlsxwriter -3.2.9. +fixture the .NET, C++ and Rust suites use. Measured on Windows 10 (22H2), AMD Ryzen 7 5700X +(16 logical CPUs @ 3.39 GHz), CPython 3.14.7, 10 runs each (medians shown; `min` is in the scripts' own +output), against a library published from commit 65945b4. Competitors: polars 1.44.1 (with +fastexcel 0.21.0), pandas 3.0.5 (with openpyxl 3.1.5), xlsxwriter 3.2.9. ### Reading | API | Median | What it produces | |---|---:|---| -| `to_record_batch_reader()` | 35.8 ms | Arrow record batches, streamed | -| `to_arrow()` | 36.1 ms | the same parse, handed to pyarrow zero-copy | -| `parse_typed()` | 39.5 ms | typed columnar buffers, converted natively | -| `iter_parse_typed()` | 38.4 ms | the same typed buffers, in batches | -| `to_polars()` | 38.7 ms | typed columnar DataFrame, schema inferred | -| `polars.read_excel()` | 124.3 ms | typed columnar DataFrame, types inferred | -| `read_all_columnar()` | 471.7 ms | raw columnar cells, no per-cell Python objects | -| `rows()` | 1,086.9 ms | one `Cell` object per cell, streamed per row | -| `read_all()` | 1,656.5 ms | one `Cell` object per cell, all at once | +| `to_record_batch_reader()` | 31.9 ms | Arrow record batches, streamed | +| `to_arrow()` | 31.8 ms | the same parse, handed to pyarrow zero-copy | +| `parse_typed()` | 33.5 ms | typed columnar buffers, converted natively | +| `iter_parse_typed()` | 34.2 ms | the same typed buffers, in batches | +| `to_polars()` | 34.6 ms | typed columnar DataFrame, schema inferred | +| `polars.read_excel()` | 125.6 ms | typed columnar DataFrame, types inferred | +| `read_all_columnar()` | 463.3 ms | raw columnar cells, no per-cell Python objects | +| `rows()` | 1,034.6 ms | one `Cell` object per cell, streamed per row | +| `read_all()` | 1,631.0 ms | one `Cell` object per cell, all at once | The batched readers cost the same as their whole-sheet counterparts and lower the peak memory: -`to_record_batch_reader(batch_size=10000)` peaks at 20.2 MiB against 24.8 MiB for one whole-sheet +`to_record_batch_reader(batch_size=10000)` peaks at 11.4 MiB against 18.8 MiB for one whole-sheet record batch on this file. -Only the `to_polars()` / `polars.read_excel()` pair is a like-for-like comparison, and even that one -is loose: both produce a typed columnar DataFrame with inferred types, but the inference rules are -not identical. The rows above it produce different things and are listed to show what each API -costs, not to rank them — `read_all()` is ~42x slower than `parse_typed()` because it materializes +Only the `to_polars()` / `polars.read_excel()` pair is a like-for-like comparison (~3.6x), and even +that one is loose: both produce a typed columnar DataFrame with inferred types, but the inference rules +are not identical. The rows above it produce different things and are listed to show what each API +costs, not to rank them — `read_all()` is ~49x slower than `parse_typed()` because it materializes 917,504 Python objects, which is the price of that shape, not a slow parser. +CSV, from `65K_Records_Data.csv` (8.2 MB), same 14 typed columns: + +| API | Median | +|---|---:| +| `parse_typed(parallelism=1)` | 23.6 ms | +| `parse_typed(parallelism=0)` | 12.0 ms | +| `to_polars(parallelism=1)` | 25.0 ms | +| `to_polars(parallelism=0)` | 10.8 ms | +| `polars.read_csv(try_parse_dates=True)` | 7.3 ms | +| `pandas.read_csv`, pyarrow engine | 115.8 ms | +| `pandas.read_csv`, C engine | 138.7 ms | + +polars' CSV reader is ahead (multithreaded, and it infers types from the text without our schema +step); ExcelReader is ~10x faster than pandas on either engine. These include opening the file, so +they are above the in-memory `xl_parse_typed` figures in [Reading a large sheet on several +threads](#reading-a-large-sheet-on-several-threads). + ### Writing | API | Median | Output | |---|---:|---| -| `write_workbook()` → csv | 35.7 ms | 8.2 MB | -| `write_workbook()` → xls | 43.9 ms | 17.7 MB | -| `write_workbook()` → xlsb | 71.9 ms | 5.1 MB | -| `write_workbook()` → xlsx | 90.2 ms | 5.1 MB | -| `write_polars()` → xlsx | 470.2 ms | 5.1 MB | -| `write_pandas()` → xlsx | 468.5 ms | 5.1 MB | -| `polars.DataFrame.write_excel()` | 4,707.2 ms | 5.6 MB | -| `pandas.DataFrame.to_excel()` | 7,274.0 ms | 5.5 MB | +| `write_workbook()` → csv | 35.1 ms | 8.2 MB | +| `write_workbook()` → xls | 44.4 ms | 17.7 MB | +| `write_workbook()` → xlsb | 71.5 ms | 5.1 MB | +| `write_workbook()` → xlsx | 90.6 ms | 5.1 MB | +| `write_polars()` → xlsx | 487.4 ms | 5.1 MB | +| `write_pandas()` → xlsx | 493.8 ms | 5.1 MB | +| `polars.DataFrame.write_excel()` | 4,670.4 ms | 5.6 MB | +| `pandas.DataFrame.to_excel()` | 7,192.9 ms | 5.5 MB | The two DataFrame comparisons are matched work — same DataFrame in, xlsx out both times: -`write_polars()` is ~10.0x faster than polars' own `write_excel()`, and `write_pandas()` ~15.5x +`write_polars()` is ~9.6x faster than polars' own `write_excel()`, and `write_pandas()` ~14.6x faster than `to_excel()`. Both of ours pay a conversion the raw path does not: the DataFrame goes through Arrow and then a Python list before reaching the native columns, which is most of the gap -between the 470 ms row and the 90 ms one. Handing `write_workbook()` buffers that are already +between the ~490 ms rows and the 90 ms one. Handing `write_workbook()` buffers that are already columnar — what `parse_typed()` returns — skips all of it. -`write_workbook(xlsx)` at 90.2 ms lands within ~5 ms of the C++ binding's -`xl::write_columns` on the same 14 columns (85.1–85.9 ms), which is the expected result: both are thin wrappers +`write_workbook(xlsx)` at 90.6 ms lands within ~5 ms of the C++ binding's +`xl::write_columns` on the same 14 columns (85.4 ms), which is the expected result: both are thin wrappers over the same `xl_write_typed` call, and neither adds work per cell. `xls` being both the largest file and faster than xlsb and xlsx is not a paradox — BIFF8 writes diff --git a/python/scripts/api_markdown.py b/python/scripts/api_markdown.py new file mode 100644 index 00000000..027fd0b7 --- /dev/null +++ b/python/scripts/api_markdown.py @@ -0,0 +1,223 @@ +"""Write the Python API reference as Markdown pages plus a toc.yml, for docfx to render. + +Run from anywhere: python python/scripts/api_markdown.py +""" + +import ast +import enum +import inspect +import re +import sys +import typing +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +OUTPUT_DIR = REPO_ROOT / "docs" / "python-api" +MAX_LINE = 88 + +sys.path.insert(0, str(REPO_ROOT / "python" / "src")) +import excelreader # type: ignore[import-not-found] # noqa: E402 + +LINKABLE = {name for name in excelreader.__all__ if inspect.isclass(getattr(excelreader, name))} + +PROTOCOLS = { + "__len__": "`len()`", + "__getitem__": "indexing", + "__iter__": "iteration", + "__contains__": "`in`", + "__enter__": "`with`", +} + + +class _Raw(str): + """An annotation kept as written, so a signature prints `int | None` rather than `'int | None'`.""" + + def __repr__(self) -> str: + return str(self) + + +def _raw(annotation: object) -> object: + annotation = getattr(annotation, "__forward_arg__", annotation) + return _Raw(annotation) if isinstance(annotation, str) else annotation + + +def signature(function: typing.Callable, drop_first: bool = False) -> str: + sig = inspect.signature(function) + parameters = [p.replace(annotation=_raw(p.annotation)) for p in sig.parameters.values()] + if drop_first: + parameters = parameters[1:] + sig = sig.replace(parameters=parameters, return_annotation=_raw(sig.return_annotation)) + one_line = f"def {function.__name__}{sig}" + if len(one_line) <= MAX_LINE: + return one_line + + lines = [] + for index, parameter in enumerate(parameters): + previous = parameters[index - 1].kind if index else None + if previous is inspect.Parameter.POSITIONAL_ONLY and parameter.kind is not previous: + lines.append("/") + if parameter.kind is inspect.Parameter.KEYWORD_ONLY and previous not in ( + inspect.Parameter.KEYWORD_ONLY, + inspect.Parameter.VAR_POSITIONAL, + ): + lines.append("*") + lines.append(str(parameter)) + if parameters and parameters[-1].kind is inspect.Parameter.POSITIONAL_ONLY: + lines.append("/") + returns = "" if sig.return_annotation is inspect.Signature.empty else f" -> {sig.return_annotation}" + body = "".join(f" {line},\n" for line in lines) + return f"def {function.__name__}(\n{body}){returns}" + + +def signatures(function: typing.Callable, drop_first: bool = False) -> str: + overloads = typing.get_overloads(function) or [function] + rendered = [signature(f, drop_first) for f in overloads] + for text in rendered: + ast.parse(f"{text}: ...") + return "\n".join(rendered) + + +def doc(obj: object) -> str: + return inspect.getdoc(obj) or "" + + +def cell(text: object) -> str: + return str(text).replace("|", "\\|") + + +def linked(annotation: object) -> str: + text = cell(_raw(annotation)).replace("[", "\\[").replace("]", "\\]") + return re.sub( + r"\b[A-Za-z_]\w*\b", + lambda m: f"[{m[0]}]({m[0]}.md)" if m[0] in LINKABLE else m[0], + text, + ) + + +def parameters_and_returns(function: typing.Callable, drop_first: bool) -> str: + sig = inspect.signature(function) + parameters = list(sig.parameters.values())[1 if drop_first else 0 :] + parts = [] + if parameters: + rows = "\n".join( + f"| `{p.name}` | " + + ("" if p.annotation is inspect.Parameter.empty else linked(p.annotation)) + + " | " + + ("" if p.default is inspect.Parameter.empty else f"`{cell(repr(p.default))}`") + + " |" + for p in parameters + ) + parts.append(f"**Parameters**\n\n| Name | Type | Default |\n|---|---|---|\n{rows}") + if sig.return_annotation not in (inspect.Signature.empty, "None", None): + parts.append(f"**Returns** {linked(sig.return_annotation)}") + return "\n\n".join(parts) + + +def code(text: str) -> str: + return f"```python\n{text}\n```" + + +def bases(cls: type) -> str: + names = [getattr(b, "__name__", str(b)) for b in getattr(cls, "__orig_bases__", cls.__bases__)] + names = [n for n in names if n != "object"] + return f"({', '.join(names)})" if names else "" + + +def class_page(cls: type) -> str: + kind = "Enum" if issubclass(cls, enum.Enum) else "Exception" if issubclass(cls, BaseException) else "Class" + parts = [f"# {kind} {cls.__name__}", doc(cls), code(f"class {cls.__name__}{bases(cls)}")] + + inherited = [b.__name__ for b in cls.__bases__ if b.__name__ in LINKABLE] + if inherited: + parts.append("**Inherits** " + ", ".join(linked(name) for name in inherited)) + + supported = [label for name, label in PROTOCOLS.items() if name in vars(cls)] + if supported: + parts.append(f"Supports {', '.join(supported)}.") + + if issubclass(cls, enum.Enum): + rows = "\n".join(f"| `{m.name}` | {cell(m.value)} |" for m in cls) + parts.append(f"## Members\n\n| Name | Value |\n|---|---|\n{rows}") + elif hasattr(cls, "_fields"): + annotations = inspect.get_annotations(cls) + defaults = getattr(cls, "_field_defaults", {}) + rows = "\n".join( + f"| `{f}` | {linked(annotations.get(f, ''))} | " + + (f"`{cell(repr(defaults[f]))}`" if f in defaults else "") + + " |" + for f in cls._fields + ) + parts.append(f"## Fields\n\n| Name | Type | Default |\n|---|---|---|\n{rows}") + + members = [(n, v) for n, v in vars(cls).items() if not n.startswith("_")] + properties = [(n, v) for n, v in members if isinstance(v, property)] + methods = [(n, v) for n, v in members if inspect.isfunction(v) or isinstance(v, (staticmethod, classmethod))] + + if properties: + parts.append("## Properties") + for name, prop in properties: + returns = inspect.signature(prop.fget).return_annotation if prop.fget else inspect.Signature.empty + typed = "" if returns is inspect.Signature.empty else f": {returns}" + parts += [f"### {name}", code(f"{name}{typed}")] + if "](" in linked(returns): + parts.append(f"**Type** {linked(returns)}") + parts.append(doc(prop)) + + if methods: + parts.append("## Methods") + for name, method in methods: + function: typing.Callable = method.__func__ if isinstance(method, (staticmethod, classmethod)) else method + drop_first = not isinstance(method, staticmethod) + parts += [ + f"### {name}", + code(signatures(function, drop_first)), + doc(function), + parameters_and_returns(function, drop_first), + ] + + return "\n\n".join(p for p in parts if p) + "\n" + + +def functions_page(functions: list) -> str: + parts = ["# Functions"] + for function in functions: + parts += [ + f"## {function.__name__}", + code(signatures(function)), + doc(function), + parameters_and_returns(function, drop_first=False), + ] + return "\n\n".join(p for p in parts if p) + "\n" + + +def toc_section(name: str, classes: list) -> str: + items = "".join(f" - name: {c.__name__}\n href: {c.__name__}.md\n" for c in classes) + return f"- name: {name}\n items:\n{items}" if classes else "" + + +def main() -> None: + public = [getattr(excelreader, name) for name in excelreader.__all__] + classes = [o for o in public if inspect.isclass(o)] + functions = [o for o in public if inspect.isfunction(o)] + enums = [c for c in classes if issubclass(c, enum.Enum)] + exceptions = [c for c in classes if issubclass(c, BaseException)] + others = [c for c in classes if c not in enums and c not in exceptions] + + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + for stale in OUTPUT_DIR.iterdir(): + stale.unlink() + for cls in classes: + (OUTPUT_DIR / f"{cls.__name__}.md").write_text(class_page(cls), encoding="utf-8") + (OUTPUT_DIR / "functions.md").write_text(functions_page(functions), encoding="utf-8") + + toc = ( + toc_section("Classes", others) + + toc_section("Enums", enums) + + toc_section("Exceptions", exceptions) + + "- name: Functions\n href: functions.md\n" + ) + (OUTPUT_DIR / "toc.yml").write_text(toc, encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/python/scripts/build_native.py b/python/scripts/build_native.py index adaa3fe4..c85cf58d 100644 --- a/python/scripts/build_native.py +++ b/python/scripts/build_native.py @@ -4,6 +4,7 @@ """ import argparse +import os import platform import shutil import subprocess @@ -32,6 +33,17 @@ def default_rid() -> str: return f"linux-{arch}" +def publish_environment() -> dict[str, str]: + """The NativeAOT linker step finds MSVC through vswhere, which the installer keeps off PATH.""" + environment = dict(os.environ) + if platform.system() == "Windows" and shutil.which("vswhere") is None: + program_files = environment.get("ProgramFiles(x86)", r"C:\Program Files (x86)") + installer = Path(program_files) / "Microsoft Visual Studio" / "Installer" + if (installer / "vswhere.exe").exists(): + environment["PATH"] = environment.get("PATH", "") + os.pathsep + str(installer) + return environment + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--rid", default=default_rid(), help="dotnet runtime identifier") @@ -41,6 +53,7 @@ def main() -> int: subprocess.run( ["dotnet", "publish", str(CSPROJ), "-c", "Release", "-f", args.framework, "-r", args.rid], check=True, + env=publish_environment(), ) publish_dir = CSPROJ.parent / "bin" / "Release" / args.framework / args.rid / "publish" diff --git a/python/src/excelreader/__init__.py b/python/src/excelreader/__init__.py index b5cf6ba4..6b8457d1 100644 --- a/python/src/excelreader/__init__.py +++ b/python/src/excelreader/__init__.py @@ -2,6 +2,7 @@ from excelreader.arrow_stream import ArrowStream from excelreader.reader import Sheet, Sheets, Workbook, decode_cell, open_bytes, open_workbook +from excelreader.sources import Source, open_source, open_stream from excelreader.types import ( Cell, CellType, @@ -46,6 +47,7 @@ "SheetVisibility", "SheetWriter", "Sheets", + "Source", "StringColumn", "TypedTable", "Workbook", @@ -54,6 +56,8 @@ "encrypt_package", "encrypt_package_bytes", "open_bytes", + "open_source", + "open_stream", "open_workbook", "open_writer", "open_writer_to_memory", diff --git a/python/src/excelreader/_native.py b/python/src/excelreader/_native.py index a619aacb..c67def55 100644 --- a/python/src/excelreader/_native.py +++ b/python/src/excelreader/_native.py @@ -263,6 +263,37 @@ class NativeOpenOptions(ctypes.Structure): ("intern_strings", ctypes.c_int32), ("password", ctypes.POINTER(ctypes.c_uint8)), ("password_len", ctypes.c_int32), + ("source_cache_bytes", ctypes.c_int64), + ("max_buffered_bytes", ctypes.c_int64), + ("source_block_size", ctypes.c_int32), + ] + + +SourceReadAt = ctypes.CFUNCTYPE(ctypes.c_int64, ctypes.c_void_p, ctypes.c_int64, ctypes.c_void_p, ctypes.c_int64) +StreamRead = ctypes.CFUNCTYPE(ctypes.c_int64, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_int64) +SourceRelease = ctypes.CFUNCTYPE(None, ctypes.c_void_p) + + +class NativeSource(ctypes.Structure): + """Mirrors xl_source.""" + + _fields_ = [ + ("struct_size", ctypes.c_int32), + ("user_data", ctypes.c_void_p), + ("length", ctypes.c_int64), + ("read_at", SourceReadAt), + ("release", SourceRelease), + ] + + +class NativeStream(ctypes.Structure): + """Mirrors xl_stream.""" + + _fields_ = [ + ("struct_size", ctypes.c_int32), + ("user_data", ctypes.c_void_p), + ("read", StreamRead), + ("release", SourceRelease), ] @@ -310,6 +341,9 @@ def to_native_open_options(options: OpenOptions) -> NativeOpenOptions: raw.max_cell_bytes = _opt_number(options.max_cell_bytes) raw.max_shared_string_bytes = _opt_number(options.max_shared_string_bytes) raw.max_zip_entries = _opt_number(options.max_zip_entries) + raw.source_cache_bytes = _opt_number(options.source_cache_bytes) + raw.max_buffered_bytes = _opt_number(options.max_buffered_bytes) + raw.source_block_size = _opt_number(options.source_block_size) return raw @@ -403,8 +437,6 @@ def load_library() -> ctypes.CDLL: def _warm_up(lib: ctypes.CDLL) -> None: - # Threads whose first native call lands together can crash the NativeAOT runtime's start-up - # (see commit b5f8c5f); one small read here finishes it on a single thread. Failures are ignored. data = b"a,b\n1,2\n3,4\n" workbook = ctypes.c_void_p() if lib.xl_open_memory(data, len(data), XL_FORMAT_CSV, None, ctypes.byref(workbook)) != XL_OK: @@ -445,6 +477,12 @@ def _bind(lib: ctypes.CDLL) -> ctypes.CDLL: lib.xl_open_file.restype = c_int lib.xl_open_memory.argtypes = [p_bytes, c_int, c_int, p_open_options, pp_void] lib.xl_open_memory.restype = c_int + lib.xl_open_source.argtypes = [ctypes.POINTER(NativeSource), c_int, p_open_options, pp_void] + lib.xl_open_source.restype = c_int + lib.xl_open_stream.argtypes = [ctypes.POINTER(NativeStream), c_int, p_open_options, pp_void] + lib.xl_open_stream.restype = c_int + lib.xl_set_source_error.argtypes = [p_bytes, c_int] + lib.xl_set_source_error.restype = None lib.xl_sheet_visibility_at.argtypes = [p_void, c_int, p_int] lib.xl_sheet_visibility_at.restype = c_int lib.xl_sheet_index.argtypes = [p_void, p_bytes, c_int, p_int] diff --git a/python/src/excelreader/arrow_stream.py b/python/src/excelreader/arrow_stream.py index bd3c0399..3292e25e 100644 --- a/python/src/excelreader/arrow_stream.py +++ b/python/src/excelreader/arrow_stream.py @@ -17,7 +17,60 @@ _capsule_pointer.argtypes = [ctypes.c_void_p, ctypes.c_char_p] _capsule_pointer.restype = ctypes.c_void_p -_live: dict[int, _native.ArrowArrayStream] = {} +_ReadFn = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p) +_ErrorFn = ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p) +_ReleaseFn = ctypes.CFUNCTYPE(None, ctypes.c_void_p) + + +class _Target: + """The native stream behind a proxy, with its callbacks re-typed so ctypes releases the GIL.""" + + def __init__(self, stream: _native.ArrowArrayStream) -> None: + self.stream = stream + self.address = ctypes.addressof(stream) + self.get_schema = _ReadFn(_address_of(stream.get_schema)) + self.get_next = _ReadFn(_address_of(stream.get_next)) + self.get_last_error = _ErrorFn(_address_of(stream.get_last_error)) + + +def _address_of(function: object) -> int: + return ctypes.cast(function, ctypes.c_void_p).value + + +_streams: dict[int, _Target] = {} +_proxies: dict[int, _native.ArrowArrayStream] = {} + + +def _target(stream: int) -> _Target: + return _streams[_native.ArrowArrayStream.from_address(stream).private_data] + + +@_ReadFn +def _get_schema(stream: int, out: int) -> int: + target = _target(stream) + return target.get_schema(target.address, out) + + +@_ReadFn +def _get_next(stream: int, out: int) -> int: + target = _target(stream) + return target.get_next(target.address, out) + + +@_ErrorFn +def _get_last_error(stream: int) -> int | None: + target = _target(stream) + return target.get_last_error(target.address) + + +def _close(proxy: _native.ArrowArrayStream) -> None: + _release(_streams.pop(proxy.private_data).stream) + proxy.release = _native._ArrowStreamRelease() + + +@_ReleaseFn +def _release_proxy(stream: int) -> None: + _close(_native.ArrowArrayStream.from_address(stream)) def _release(stream: _native.ArrowArrayStream) -> None: @@ -25,10 +78,22 @@ def _release(stream: _native.ArrowArrayStream) -> None: stream.release(ctypes.byref(stream)) +def _proxy(stream: _native.ArrowArrayStream) -> _native.ArrowArrayStream: + target = _Target(stream) + _streams[id(target)] = target + proxy = _native.ArrowArrayStream() + proxy.get_schema = ctypes.cast(_get_schema, _native._ArrowStreamGetSchema) + proxy.get_next = ctypes.cast(_get_next, _native._ArrowStreamGetNext) + proxy.get_last_error = ctypes.cast(_get_last_error, _native._ArrowStreamGetLastError) + proxy.release = ctypes.cast(_release_proxy, _native._ArrowStreamRelease) + proxy.private_data = id(target) + return proxy + + def _destroy_capsule(capsule: int) -> None: - address = _capsule_pointer(capsule, _CAPSULE_NAME) - stream = _live.pop(address) - _release(stream) + proxy = _proxies.pop(_capsule_pointer(capsule, _CAPSULE_NAME)) + if proxy.release: + _close(proxy) _destroy_capsule_callback = ctypes.CFUNCTYPE(None, ctypes.c_void_p)(_destroy_capsule) @@ -49,8 +114,9 @@ def __arrow_c_stream__(self, requested_schema: object = None) -> object: if self._stream is None: raise ExcelReaderError("this Arrow stream was already consumed") stream, self._stream = self._stream, None - address = ctypes.addressof(stream) - _live[address] = stream + proxy = _proxy(stream) + address = ctypes.addressof(proxy) + _proxies[address] = proxy return _capsule_new(address, _CAPSULE_NAME, _destroy_capsule_callback) def __del__(self) -> None: diff --git a/python/src/excelreader/sources.py b/python/src/excelreader/sources.py new file mode 100644 index 00000000..faf39835 --- /dev/null +++ b/python/src/excelreader/sources.py @@ -0,0 +1,119 @@ +"""Open a workbook from bytes the caller serves: a positional `Source`, or a readable stream.""" + +from __future__ import annotations + +import ctypes +import itertools +import operator +from typing import Protocol + +from excelreader import _native +from excelreader.reader import Workbook, _check, _raw_options, _resolve_format +from excelreader.types import OpenOptions + + +class Source(Protocol): + """Random-access bytes for `open_source()`. `read_at` is called from several threads at once.""" + + size: int + + def read_at(self, offset: int, buffer: memoryview) -> int: + """Fills `buffer` with the bytes starting at `offset` and returns how many it wrote. + + `buffer` is valid only during the call; do not retain it. + """ + ... + + +_keys = itertools.count(1) +_live: dict[int, object] = {} + + +def _fail(error: BaseException) -> int: + message = (str(error) or type(error).__name__).encode("utf-8", errors="replace") + _native.load_library().xl_set_source_error(message, len(message)) + return -1 + + +def _view(address: int, length: int) -> memoryview: + return memoryview((ctypes.c_ubyte * length).from_address(address)).cast("B") + + +@_native.SourceReadAt +def _read_at(key: int, offset: int, buffer: int, length: int) -> int: + try: + return operator.index(_live[key].read_at(offset, _view(buffer, length))) + except BaseException as error: # noqa: BLE001 + return _fail(error) + + +@_native.StreamRead +def _read(key: int, buffer: int, length: int) -> int: + try: + stream = _live[key] + readinto = getattr(stream, "readinto", None) + if readinto is None: + data = bytes(stream.read(length)) + if len(data) > length: + raise OSError("the stream returned more bytes than requested") + ctypes.memmove(buffer, data, len(data)) + return len(data) + count = readinto(_view(buffer, length)) + if count is None: + raise OSError("the stream has no data ready; open_stream() needs a blocking stream") + count = operator.index(count) + if count > length: + raise OSError("the stream returned more bytes than requested") + return count + except BaseException as error: # noqa: BLE001 + return _fail(error) + + +@_native.SourceRelease +def _release(key: int) -> None: + _live.pop(key, None) + + +def open_source( + source: Source, format: str | None = None, *, options: OpenOptions | None = None, password: str | None = None +) -> Workbook: + """Opens a workbook over bytes `source` serves; every sheet can be read on its own thread. + + The library keeps a reference to `source` until the workbook and everything read from it are + closed. `read_at` runs with the GIL held, so a local file is better opened with `open_workbook()`. + """ + size = operator.index(source.size) + if size < 0: + raise ValueError("source.size must not be negative") + lib = _native.load_library() + resolved = _resolve_format(format, None) + raw_options, _raw, _password_buffer = _raw_options(options, password) + key = next(_keys) + raw = _native.NativeSource(ctypes.sizeof(_native.NativeSource), key, size, _read_at, _release) + handle = ctypes.c_void_p() + _live[key] = source + _check(lib.xl_open_source(ctypes.byref(raw), resolved, raw_options, ctypes.byref(handle))) + return Workbook(handle) + + +def open_stream( + stream: object, format: str | None = None, *, options: OpenOptions | None = None, password: str | None = None +) -> Workbook: + """Opens a workbook over a stream that cannot seek, such as `sys.stdin.buffer` or a socket. + + A CSV (`format="csv"`, or no format and neither a ZIP nor an OLE2 signature) is read as it + arrives, and can be read again from the start + (`infer_schema()`, then a read) until 16 MiB of it were read; an XLSX, XLSB or XLS is read whole + before the workbook opens. `stream` needs `readinto` or `read`; the library never closes it. + """ + if not (hasattr(stream, "readinto") or hasattr(stream, "read")): + raise TypeError("open_stream() needs an object with readinto() or read()") + lib = _native.load_library() + resolved = _resolve_format(format, None) + raw_options, _raw, _password_buffer = _raw_options(options, password) + key = next(_keys) + raw = _native.NativeStream(ctypes.sizeof(_native.NativeStream), key, _read, _release) + handle = ctypes.c_void_p() + _live[key] = stream + _check(lib.xl_open_stream(ctypes.byref(raw), resolved, raw_options, ctypes.byref(handle))) + return Workbook(handle) diff --git a/python/src/excelreader/types.py b/python/src/excelreader/types.py index 8693c8c5..c562a61a 100644 --- a/python/src/excelreader/types.py +++ b/python/src/excelreader/types.py @@ -117,6 +117,10 @@ class OpenOptions(NamedTuple): `csv_delimiter` and `csv_quote` are byte values, not strings: pass `ord(';')`, not `';'`. + `source_block_size` and `source_cache_bytes` tune the block cache of `open_source()` (4 MiB blocks + in 64 MiB by default; a negative block size turns the cache off). `max_buffered_bytes` caps how + much `open_stream()` reads before an XLSX, XLSB or XLS opens. + Validation happens on the native side, which owns the real limits; an out-of-range value raises `ExcelReaderError` with the reason. Fields mirror xl_open_options in src/ExcelReader.Native/include/excelreader.h. @@ -134,6 +138,9 @@ class OpenOptions(NamedTuple): max_zip_entries: int | None = None prefetch_decompression: bool | None = None intern_strings: bool | None = None + source_block_size: int | None = None + source_cache_bytes: int | None = None + max_buffered_bytes: int | None = None class WriteOptions(NamedTuple): diff --git a/python/tests/test_native.py b/python/tests/test_native.py index 735934f1..291186b6 100644 --- a/python/tests/test_native.py +++ b/python/tests/test_native.py @@ -22,6 +22,9 @@ def test_exported_functions_are_present(): for name in ( "xl_open_file", "xl_open_memory", + "xl_open_source", + "xl_open_stream", + "xl_set_source_error", "xl_close", "xl_sheet_count", "xl_sheet_name_at", diff --git a/python/tests/test_reader.py b/python/tests/test_reader.py index ad3a6476..1f9888ba 100644 --- a/python/tests/test_reader.py +++ b/python/tests/test_reader.py @@ -1,5 +1,9 @@ +import ctypes import gc import shutil +import sys +import threading +import time from pathlib import Path import pytest @@ -16,6 +20,7 @@ open_bytes, open_workbook, ) +from excelreader import _native from excelreader import arrow_stream as arrow_stream_module from excelreader import reader as reader_module @@ -649,6 +654,51 @@ def test_a_dropped_capsule_releases_the_read(batched_csv, monkeypatch): assert released == [True] +def test_a_consumer_holding_the_gil_does_not_block_other_threads_while_a_batch_parses(tmp_path): + path = tmp_path / "large.csv" + rows = 1_000_000 + path.write_text("name,qty\n" + "".join(f"row{i},{i}\n" for i in range(rows)), encoding="utf-8") + ticks = [] + stop = threading.Event() + + def tick(): + while not stop.is_set(): + ticks.append(time.perf_counter()) + + with open_workbook(path) as workbook: + capsule = workbook.sheets[0].to_arrow_stream(_BATCH_SCHEMA, batch_size=rows).__arrow_c_stream__() + capsule_pointer = ctypes.PYFUNCTYPE(ctypes.c_void_p, ctypes.py_object, ctypes.c_char_p)( + ("PyCapsule_GetPointer", ctypes.pythonapi) + ) + address = capsule_pointer(capsule, b"arrow_array_stream") + stream = _native.ArrowArrayStream.from_address(address) + holding_gil = ctypes.PYFUNCTYPE(ctypes.c_int, ctypes.c_void_p, ctypes.c_void_p) + get_next = holding_gil(ctypes.cast(stream.get_next, ctypes.c_void_p).value) + batch = _native.ArrowArray() + switch_interval = sys.getswitchinterval() + sys.setswitchinterval(0.001) + ticker = threading.Thread(target=tick) + ticker.start() + try: + start = time.perf_counter() + status = get_next(address, ctypes.addressof(batch)) + end = time.perf_counter() + finally: + stop.set() + ticker.join() + sys.setswitchinterval(switch_interval) + release_batch = ctypes.CFUNCTYPE(None, ctypes.c_void_p)(batch.release) + release_batch(ctypes.addressof(batch)) + del capsule + + assert status == 0 + assert batch.length == rows + inside = [start] + [t for t in ticks if start < t < end] + [end] + longest_gap = max(b - a for a, b in zip(inside, inside[1:])) + assert end - start > 0.02, "the batch must parse for many switch intervals for the gap to mean something" + assert longest_gap < (end - start) / 2 + + def test_arrow_stream_rejects_a_negative_batch_size(batched_csv): with open_workbook(batched_csv) as workbook: with pytest.raises(ExcelReaderError): diff --git a/python/tests/test_sources.py b/python/tests/test_sources.py new file mode 100644 index 00000000..a8353732 --- /dev/null +++ b/python/tests/test_sources.py @@ -0,0 +1,209 @@ +import gc +import io +import os +import threading +import weakref +from concurrent.futures import ThreadPoolExecutor + +import pytest +from excelreader import ExcelReaderError, OpenOptions, open_bytes, open_source, open_stream, open_writer_to_memory +from excelreader import sources as sources_module + +SHEETS = 4 +ROWS = 300 + + +def _workbook_bytes() -> bytes: + with open_writer_to_memory("xlsx") as writer: + for sheet in range(SHEETS): + writer.start_sheet(f"sheet{sheet}") + for row in range(ROWS): + writer.write_row([f"s{sheet}-r{row}"]) + writer.end_sheet() + return writer.bytes() + + +def _first_column(sheet) -> list[str]: + return [row[0].value for row in sheet.rows()] + + +class BytesSource: + def __init__(self, data: bytes) -> None: + self.data = data + self.size = len(data) + self.reads = 0 + self._lock = threading.Lock() + + def read_at(self, offset: int, buffer: memoryview) -> int: + with self._lock: + self.reads += 1 + chunk = self.data[offset : offset + len(buffer)] + buffer[: len(chunk)] = chunk + return len(chunk) + + +class FailingSource: + size = 1024 + + def read_at(self, offset: int, buffer: memoryview) -> int: + raise OSError("disk on fire") + + +class OnlyRead: + def __init__(self, data: bytes) -> None: + self._inner = io.BytesIO(data) + + def read(self, size: int) -> bytes: + return self._inner.read(min(size, 7)) + + +def test_every_sheet_of_a_source_reads_in_parallel_like_memory(): + data = _workbook_bytes() + with open_bytes(data) as reference: + expected = [_first_column(sheet) for sheet in reference.sheets] + + with open_source(BytesSource(data)) as workbook, ThreadPoolExecutor(max_workers=SHEETS * 2) as pool: + sheets = list(workbook.sheets) + parallel = list(pool.map(_first_column, sheets + sheets)) + + for position, rows in enumerate(parallel): + assert rows == expected[position % SHEETS] + + +def test_the_source_is_dropped_after_the_workbook_closes(): + source = BytesSource(_workbook_bytes()) + finalized = weakref.finalize(source, lambda: None) + before = set(sources_module._live) + workbook = open_source(source, "xlsx") + assert len(sources_module._live) == len(before) + 1 + del source + _first_column(workbook.sheets[0]) + workbook.close() + gc.collect() + assert not finalized.alive + assert set(sources_module._live) == before + + +def test_a_source_error_reaches_the_caller_with_its_message(): + with pytest.raises(ExcelReaderError, match="disk on fire"): + open_source(FailingSource(), "xlsx") + + +def test_the_default_cache_fetches_a_small_workbook_once(): + data = _workbook_bytes() + cached = BytesSource(data) + with open_source(cached, "xlsx") as workbook: + _first_column(workbook.sheets[0]) + assert cached.reads == 1 + + uncached = BytesSource(data) + with open_source(uncached, "xlsx", options=OpenOptions(source_block_size=-1)) as workbook: + _first_column(workbook.sheets[0]) + assert uncached.reads > 1 + + +def test_one_source_object_opened_twice_at_once_serves_both(): + source = BytesSource(_workbook_bytes()) + first = open_source(source, "xlsx") + second = open_source(source, "xlsx") + first.close() + assert _first_column(second.sheets[2])[0] == "s2-r0" + second.close() + + +def test_open_stream_reads_a_csv_from_a_pipe(): + read_fd, write_fd = os.pipe() + + def produce() -> None: + with os.fdopen(write_fd, "wb") as out: + out.write(b"name,qty\n") + for i in range(1000): + out.write(f"row{i},{i}\n".encode()) + + producer = threading.Thread(target=produce) + producer.start() + with os.fdopen(read_fd, "rb") as pipe, open_stream(pipe, "csv") as workbook: + names = _first_column(workbook.sheets[0]) + producer.join() + assert names[0] == "name" + assert len(names) == 1001 + assert names[-1] == "row999" + + +def test_a_csv_stream_serves_a_schema_inference_and_then_every_row(): + data = b"name,qty\n" + b"".join(f"row{i},{i}\n".encode() for i in range(5000)) + with open_stream(OnlyRead(data), "csv") as workbook: + sheet = workbook.sheets[0] + schema = sheet.infer_schema(parse_text=True) + names = _first_column(sheet) + assert len(schema) == 2 + assert len(names) == 5001 + assert names[-1] == "row4999" + + +def test_open_stream_reads_an_xlsx_from_an_object_with_only_read(): + with open_stream(OnlyRead(_workbook_bytes())) as workbook: + rows = _first_column(workbook.sheets[3]) + assert rows[0] == "s3-r0" + assert len(rows) == ROWS + + +def test_open_stream_rejects_an_object_that_cannot_be_read(): + with pytest.raises(TypeError): + open_stream(object()) + + +def test_a_failed_open_leaves_nothing_registered(): + before = set(sources_module._live) + source = BytesSource(_workbook_bytes()) + with pytest.raises(ValueError): + open_source(source, "nope") + with pytest.raises(ValueError): + open_source(source, "xlsx", password="\ud800") + negative = BytesSource(b"") + negative.size = -1 + with pytest.raises(ValueError): + open_source(negative, "xlsx") + with pytest.raises(ValueError): + open_stream(io.BytesIO(), "nope") + assert set(sources_module._live) == before + + +def test_open_stream_fails_on_a_text_stream_and_releases_it(): + before = set(sources_module._live) + with pytest.raises(ExcelReaderError), open_stream(io.StringIO("a,b\n1,2\n"), "csv") as workbook: + _first_column(workbook.sheets[0]) + assert set(sources_module._live) == before + + +def test_open_stream_fails_when_read_returns_more_than_asked(): + class Greedy: + def read(self, size: int) -> bytes: + return b"x" * (size + 1) + + before = set(sources_module._live) + with pytest.raises(ExcelReaderError, match="more bytes than requested"), open_stream(Greedy(), "csv") as workbook: + _first_column(workbook.sheets[0]) + assert set(sources_module._live) == before + + +def test_open_stream_fails_when_read_raises_keyboard_interrupt_and_releases_it(): + class Interrupted: + def read(self, size: int) -> bytes: + raise KeyboardInterrupt + + before = set(sources_module._live) + with pytest.raises(ExcelReaderError), open_stream(Interrupted(), "csv") as workbook: + _first_column(workbook.sheets[0]) + assert set(sources_module._live) == before + + +def test_a_read_at_that_returns_none_reaches_the_caller_as_an_error(): + class Bad: + size = 1024 + + def read_at(self, offset: int, buffer: memoryview): + return None + + with pytest.raises(ExcelReaderError): + open_source(Bad(), "xlsx") diff --git a/rust/excelreader/README.md b/rust/excelreader/README.md index 7727bbef..59549d99 100644 --- a/rust/excelreader/README.md +++ b/rust/excelreader/README.md @@ -1,3 +1,5 @@ +![ExcelReader logo](https://raw.githubusercontent.com/GabrielMarquezMatte/ExcelReader/master/assets/icon.png) + # excelreader (Rust) Read and write Excel/CSV workbooks via ExcelReader's native ABI: opening a workbook (from a path or @@ -11,7 +13,7 @@ parse, schema-driven writing, row-by-row decoded reads, and Arrow export. excelreader = "2.1" ``` -```rust +```rust,no_run use excelreader::workbook::{parse_sheet, ExcelMapper, Workbook}; #[derive(Default, ExcelMapper)] @@ -25,6 +27,7 @@ struct Row { let workbook = Workbook::open("book.xlsx")?; let table = parse_sheet::(workbook.sheet(0)?, 1)?; for row in table.iter() { /* ... */ } +# Ok::<(), excelreader::Error>(()) ``` `parse_sheet` takes a `Sheet`, which `Workbook::sheet` hands out (see below). @@ -54,7 +57,8 @@ excelreader = { version = "2.1", features = ["chrono"] } ### Sheets and schema inference -```rust +```rust,no_run +# use excelreader::workbook::Workbook; let workbook = Workbook::open("book.xlsx")?; for (index, name) in workbook.sheet_names()?.iter().enumerate() { println!("{index}: {name}"); @@ -64,17 +68,19 @@ for (index, name) in workbook.sheet_names()?.iter().enumerate() { for column in workbook.sheet(1)?.infer_schema(1, 100)? { println!("{:?} -> type {}", column.name, column.column_type); } +# Ok::<(), excelreader::Error>(()) ``` `infer_schema_parse_text(header_row, sample_size)` also types text cells (every CSV field, or numbers stored as text): integers, decimals, `true`/`false` and ISO dates. `parse_sheet_parallel` and `arrow::parse_arrow_parallel` take a `degree_of_parallelism` (`0` = every processor, `1` = -sequential); only CSV is split, and the result equals `parse_sheet`'s. +sequential); XLSX, XLSB and CSV are split, and the result equals `parse_sheet`'s. A workbook hands out sheets. A `Sheet` is the workbook and an index: it holds no native resource, it is `Copy`, and every read on it opens its own cursor. -```rust +```rust,no_run +# use excelreader::workbook::Workbook; let workbook = Workbook::open("report.xlsx")?; for sheet in workbook.sheets()? { @@ -87,12 +93,15 @@ while let Some(row) = rows.next_row() { let row = row?; // ... } +# Ok::<(), excelreader::Error>(()) ``` `Workbook` is `Send + Sync`, so sheets can be read in parallel over one open workbook, which loads the shared-string table once: -```rust +```rust,no_run +# use excelreader::workbook::Workbook; +# let workbook = Workbook::open("report.xlsx")?; let tables = std::thread::scope(|scope| { let handles: Vec<_> = workbook .sheets()? @@ -101,6 +110,7 @@ let tables = std::thread::scope(|scope| { .collect(); handles.into_iter().map(|handle| handle.join().expect("no panic")).collect::, _>>() })?; +# Ok::<(), excelreader::Error>(()) ``` `Workbook::open_with` takes an explicit format and `OpenOptions`; `Workbook::open_memory` reads from @@ -111,7 +121,7 @@ a byte slice. Note that format sniffing does not detect CSV - pass `XL_FORMAT_CS `OpenOptions::password` unlocks a password-protected OOXML workbook (.xlsx/.xlsb/.xlsm) through `Workbook::open_with`/`open_memory`: -```rust +```rust,no_run use excelreader::workbook::Workbook; use excelreader::{Error, OpenOptions, XL_FORMAT_AUTO}; @@ -135,31 +145,82 @@ Writing an encrypted workbook is a second step: write the plaintext package with `write_sheet`, then wrap it with `encrypt_package`, which produces an agile-encrypted (ECMA-376 4.4) CFB container - AES-256-CBC, SHA-512, 100,000 spin iterations, with a `dataIntegrity` HMAC: -```rust +```rust,no_run use excelreader::writer::{encrypt_package, write_columns}; use excelreader::XL_FORMAT_XLSX; +# use excelreader::writer::{Column, ColumnData}; +# let ids = [1i64, 2, 3]; +# let columns = [Column { name: Some("id"), data: ColumnData::I64(&ids), validity: None }]; write_columns("plain.xlsx", XL_FORMAT_XLSX, &columns, None)?; encrypt_package("plain.xlsx", "secret.xlsx", "hunter2")?; +# Ok::<(), excelreader::Error>(()) ``` `package_path` is read twice (it is not disposed or removed), so it must already be a finished file. Encryption parameters are fixed at Excel's own defaults - there are no options - and only XLSX/XLSB packages can be encrypted, matching what `Workbook::open_with`/`open_memory` can decrypt. +### Reading from your own bytes + +Implement [`Source`] to read from anything you can read at an offset. The library calls `read_at` +from several threads, fetching 4 MiB blocks it caches; every sheet can still be read in parallel. + +```rust,ignore +struct HttpSource { client: reqwest::blocking::Client, url: String, size: u64 } + +impl excelreader::Source for HttpSource { + fn size(&self) -> std::io::Result { + Ok(self.size) + } + + fn read_at(&self, offset: u64, buf: &mut [u8]) -> std::io::Result { + let range = format!("bytes={}-{}", offset, offset + buf.len() as u64 - 1); + let bytes = self.client.get(&self.url).header("Range", range).send() + .and_then(|r| r.error_for_status()) + .and_then(|r| r.bytes()) + .map_err(std::io::Error::other)?; + buf[..bytes.len()].copy_from_slice(&bytes); + Ok(bytes.len()) + } +} + +let workbook = Workbook::open_source(source, XL_FORMAT_AUTO, None)?; +``` + +Anything that implements `Read` works through `open_reader`. A CSV is read as it arrives; an XLSX, +XLSB or XLS is read whole first. + +```rust,no_run +# use excelreader::{workbook::Workbook, XL_FORMAT_CSV}; +let workbook = Workbook::open_reader(std::io::stdin(), XL_FORMAT_CSV, None)?; +# Ok::<(), excelreader::Error>(()) +``` + +An `io::Error` from your source, or a panic in it, comes back as the `Error` of the call that +needed the bytes. + ### Arrow export (`arrow` feature) ```toml excelreader = { version = "2.1", features = ["arrow"] } ``` -```rust +```rust,no_run +# #[cfg(feature = "arrow")] +# fn main() -> Result<(), excelreader::Error> { use excelreader::arrow::parse_arrow; use excelreader::workbook::Workbook; +# #[derive(Default, excelreader::workbook::ExcelMapper)] +# struct Row { #[excel(name = "Name")] name: String } let workbook = Workbook::open("book.xlsx")?; let batch = parse_arrow::(workbook.sheet(0)?, 1)?; println!("{} rows x {} columns", batch.num_rows(), batch.num_columns()); +# Ok(()) +# } +# #[cfg(not(feature = "arrow"))] +# fn main() {} ``` Off by default — arrow-rs is a large dependency and the typed-parse path needs none of it. @@ -168,9 +229,13 @@ Batched: `parse_arrow_stream` is `parse_arrow` delivered a batch at a time, retu `ArrowChunks<'_>`, an `Iterator>` that also implements `RecordBatchReader`: -```rust +```rust,no_run +# #[cfg(feature = "arrow")] +# fn main() -> Result<(), excelreader::Error> { use excelreader::arrow::parse_arrow_stream; use excelreader::workbook::Workbook; +# #[derive(Default, excelreader::workbook::ExcelMapper)] +# struct Row { #[excel(name = "Name")] name: String } let workbook = Workbook::open("book.xlsx")?; let mut chunks = parse_arrow_stream::(workbook.sheet(0)?, 1, 10_000)?; @@ -178,7 +243,10 @@ while let Some(batch) = chunks.next() { let Ok(batch) = batch else { break }; // ArrowChunks does not fuse, unlike TypedChunks println!("{} rows", batch.num_rows()); } -# Ok::<(), excelreader::Error>(()) +# Ok(()) +# } +# #[cfg(not(feature = "arrow"))] +# fn main() {} ``` Same `batch_size` semantics as `typed_chunks` below. Unlike `TypedChunks`, @@ -190,11 +258,15 @@ first `Err` rather than continuing the loop. `#[derive(ExcelMapper)]` generates both halves, so the same struct reads and writes - the field types in the table above apply unchanged: -```rust +```rust,no_run use excelreader::writer::write_sheet; use excelreader::XL_FORMAT_XLSX; +# #[derive(Default, excelreader::workbook::ExcelMapper)] +# struct Row { #[excel(name = "Name")] name: String } +# let rows = vec![Row::default()]; write_sheet("out.xlsx", XL_FORMAT_XLSX, &rows, None)?; +# Ok::<(), excelreader::Error>(()) ``` `Option` fields become an LSB-first validity bitmap: `None` writes a blank cell rather than a @@ -205,7 +277,7 @@ For buffers that are already columnar, `write_columns` borrows them and copies n lifetimes on `Column<'a>` are what turn the ABI's borrow contract into something the compiler checks: -```rust +```rust,no_run use excelreader::writer::{write_columns, Column, ColumnData}; use excelreader::XL_FORMAT_XLSX; @@ -216,6 +288,7 @@ let columns = [Column { validity: None, }]; write_columns("out.xlsx", XL_FORMAT_XLSX, &columns, None)?; +# Ok::<(), excelreader::Error>(()) ``` `validity` is checked against the row count before the call: the ABI takes the bitmap without a @@ -235,7 +308,7 @@ field. That transpose is the only copy it makes; `write_columns` pays nothing. `write_sheet`/`write_columns` build the whole table in memory first. `writer_handle::WriterHandle` is the row-by-row alternative, writing directly as each call arrives instead: -```rust +```rust,no_run use excelreader::writer_handle::WriterHandle; let mut handle = WriterHandle::open("out.xlsx", None)?; @@ -245,6 +318,7 @@ handle.write_str(Some("Name"))?; handle.write_i64(Some(42))?; handle.end_row()?; handle.end_sheet()?; +# Ok::<(), excelreader::Error>(()) ``` Call order mirrors the C ABI's `xl_writer_handle`: `open`/`open_with`/`open_memory`, then per sheet @@ -260,7 +334,7 @@ on the drop for that. `Sheet::rows` returns a cursor over that sheet. Each row borrows a buffer the cursor reuses, so iterating a sheet allocates nothing per row: -```rust +```rust,no_run use excelreader::workbook::Workbook; let workbook = Workbook::open("book.xlsx")?; @@ -279,7 +353,8 @@ Because each row borrows the cursor's buffer, only one row is alive at a time enforces it. To hold every row at once, use `read_all_decoded`, which decodes the whole sheet in one native call and keeps it alive until dropped: -```rust +```rust,no_run +# use excelreader::workbook::Workbook; let workbook = Workbook::open("book.xlsx")?; let rows = workbook.sheet(0)?.read_all_decoded()?; for row in rows.iter() { @@ -296,8 +371,10 @@ native side and never formats a cell to text. For a sheet too large to hold in memory at once, `Sheet::typed_chunks` is `parse_sheet` delivered a batch at a time: -```rust +```rust,no_run use excelreader::workbook::Workbook; +# #[derive(Default, excelreader::workbook::ExcelMapper)] +# struct Row { #[excel(name = "Name")] name: String } let workbook = Workbook::open("book.xlsx")?; let chunks = workbook.sheet(0)?.typed_chunks::(1, 10_000)?; @@ -330,6 +407,9 @@ in this crate: a .NET thread resumes at address 0 after a signal the runtime sen process with no message. It happens with any `degree_of_parallelism` above 1, and the same runtime built into a plain C host does not show it, so a Rust host appears to make it far more likely. +`parse_sheet_parallel` and `arrow::parse_arrow_parallel` now fan out on XLSX and XLSB too; this has not been tested +on macOS, so treat them the same way. + Until it is fixed upstream, pass `degree_of_parallelism: 1` on macOS for a sequential run, or keep sources under the 256 KB partitioning floor, where the library never fans out. Linux and Windows are unaffected. The crate's own partitioned aggregation tests are `#[ignore]`d on macOS for this reason. @@ -355,10 +435,10 @@ takes 20 samples instead — each of its iterations writes a whole 65,535-row fi | Benchmark | RealExcel.xlsb (100 rows) | 65K_Records_Data.xlsb (65,535 rows) | |---|---:|---:| -| `open` | 83.4 µs | 96.2 µs | -| `parse_sheet` (2 or 6 bound columns) | 60.0 µs | 29.1 ms | -| typed chunks of 1,000 / 10,000 rows | - | 29.0 ms / 27.7 ms | -| `infer_schema` (sample 100 / 1,000 rows) | 89.9 µs | 628.4 µs | +| `open` | 85.8 µs | 98.9 µs | +| `parse_sheet` (2 or 6 bound columns) | 58.7 µs | 24.6 ms | +| typed chunks of 1,000 / 10,000 rows | - | 23.4 ms / 23.5 ms | +| `infer_schema` (sample 100 / 1,000 rows) | 86.8 µs | 577.4 µs | `open` stays nearly flat across the 655x row-count jump (+15%) - XLSB's header carries its dimensions/index, so opening costs metadata, not row data. `parse_sheet` scales linearly with @@ -373,10 +453,13 @@ zero-copy advantage the other can't take: | Format | ExcelReader (`parse_sheet::`) | calamine (`worksheet_range` + `Data` match) | |---|---:|---:| -| XLSX | 85.8 ms | 272.0 ms | -| XLSB | 53.0 ms | 86.8 ms | +| XLSX | 79.1 ms | 279.7 ms | +| XLSB | 46.9 ms | 87.5 ms | + +The ExcelReader XLSX run was noisy (Criterion's interval is 76.0–83.0 ms; calamine's is 278.6–281.0 ms), so +read its ratio as 3.4–3.7x. -ExcelReader is ~3.2x faster than calamine for XLSX and ~1.6x faster for XLSB on this workload - +ExcelReader is ~3.5x faster than calamine for XLSX and ~1.9x faster for XLSB on this workload - calamine is a fast, well-optimized reader in its own right, so the gap is real but not the order of magnitude seen against slower libraries. @@ -386,15 +469,15 @@ rows to `.xlsx`, all three starting from the same in-memory `Vec`: | Benchmark | Median | |---|---:| -| `columns` (`write_columns`, pre-transposed) | 47.3 ms | -| `sheet` (`write_sheet`, from `Vec`) | 53.3 ms | -| `rust_xlsxwriter` (cell-at-a-time) | 326.9 ms | +| `columns` (`write_columns`, pre-transposed) | 45.7 ms | +| `sheet` (`write_sheet`, from `Vec`) | 51.9 ms | +| `rust_xlsxwriter` (cell-at-a-time) | 328.3 ms | `sheet` is the matched-work number — it starts from the same shape `rust_xlsxwriter` is handed and -pays the row-to-column transpose itself — and is ~6.1x faster. `columns` is a ceiling no +pays the row-to-column transpose itself — and is ~6.3x faster. `columns` is a ceiling no cell-at-a-time API can reach, since it is handed buffers that are already columnar; read it only against `sheet`, as the cost of having row-shaped data in the first place. That cost turns out to be -about 13%: not free, but a minority of the cost of producing the file. +about 14%: not free, but a minority of the cost of producing the file. Two caveats, both running against the headline number rather than for it. ExcelReader does slightly *more* work here: it attaches a number format to the date column so Excel shows a date, and writes a diff --git a/rust/excelreader/build.rs b/rust/excelreader/build.rs index 62bffe7a..07415a99 100644 --- a/rust/excelreader/build.rs +++ b/rust/excelreader/build.rs @@ -20,6 +20,11 @@ fn main() { println!("cargo:rerun-if-env-changed=EXCELREADER_NATIVE_LIB_DIR"); println!("cargo:rerun-if-changed=excelreader.def"); + // docs.rs builds offline and rustdoc never links, so the native library is not needed there. + if env::var_os("DOCS_RS").is_some() { + return; + } + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap(); let target_env = env::var("CARGO_CFG_TARGET_ENV").unwrap_or_default(); diff --git a/rust/excelreader/excelreader.def b/rust/excelreader/excelreader.def index 29729e2d..6bd570d3 100644 --- a/rust/excelreader/excelreader.def +++ b/rust/excelreader/excelreader.def @@ -2,6 +2,9 @@ EXPORTS xl_abi_version xl_open_file xl_open_memory + xl_open_source + xl_open_stream + xl_set_source_error xl_close xl_sheet_count xl_sheet_name_at diff --git a/rust/excelreader/src/lib.rs b/rust/excelreader/src/lib.rs index 9a4970d0..82e7f00c 100644 --- a/rust/excelreader/src/lib.rs +++ b/rust/excelreader/src/lib.rs @@ -1,6 +1,11 @@ -//! Raw FFI bindings to ExcelReader's C ABI (`excelreader.h`). The `extern "C"` block below is -//! declared directly in this crate root (not a submodule), so `crate::xl_open_file` etc. resolve -//! from anywhere in the crate. See the `workbook` module for the safe wrapper built on top of this. +// Raw FFI bindings to ExcelReader's C ABI (`excelreader.h`). The `extern "C"` block below is +// declared directly in this crate root (not a submodule), so `crate::xl_open_file` etc. resolve +// from anywhere in the crate. See the `workbook` module for the safe wrapper built on top of this. +#![doc = include_str!("../README.md")] +#![doc( + html_logo_url = "https://raw.githubusercontent.com/GabrielMarquezMatte/ExcelReader/master/assets/icon.png", + html_favicon_url = "https://raw.githubusercontent.com/GabrielMarquezMatte/ExcelReader/master/assets/icon.png" +)] pub mod aggregate; mod error; @@ -10,6 +15,7 @@ pub mod workbook; pub mod writer; pub mod writer_handle; pub mod rows; +pub mod source; #[cfg(feature = "arrow")] pub mod arrow; @@ -17,6 +23,7 @@ pub mod arrow; pub use aggregate::{aggregate_csv_file, aggregate_csv_memory, CsvAccumulator, CsvParallelOptions}; pub use error::Error; pub use rows::{AllRows, CellIter, CellRef, CellType, DecodedRows, RowCursor, RowRef}; +pub use source::Source; pub use options::{OpenOptions, OpenOptionsRaw, WriteOptions}; pub use temporal::{Date, Time, Timestamp}; @@ -115,9 +122,29 @@ pub struct XlOpenOptions { /// Password for an encrypted OOXML workbook, as UTF-8 bytes. NULL (with `password_len` 0) means /// "not encrypted, or fail with `XL_STATUS_PASSWORD_REQUIRED`". Not NUL-terminated - /// `password_len` is authoritative, since a password may contain any byte. The pointer need only - /// remain valid for the duration of the call - see [`OpenOptionsRaw`](crate::options::OpenOptionsRaw). + /// remain valid for the duration of the call - see [`crate::options::OpenOptionsRaw`]. pub password: *const u8, pub password_len: i32, + pub source_cache_bytes: i64, + pub max_buffered_bytes: i64, + pub source_block_size: i32, +} + +#[repr(C)] +pub struct XlSource { + pub struct_size: i32, + pub user_data: *mut c_void, + pub length: i64, + pub read_at: Option i64>, + pub release: Option, +} + +#[repr(C)] +pub struct XlStream { + pub struct_size: i32, + pub user_data: *mut c_void, + pub read: Option i64>, + pub release: Option, } /// Mirrors `xl_write_options`. Field ORDER is the C struct's, not a tidied-up version of it: with @@ -257,6 +284,22 @@ extern "C" { out_handle: *mut *mut XlWorkbook, ) -> c_int; + pub fn xl_open_source( + source: *const XlSource, + format: i32, + options: *const XlOpenOptions, + out_handle: *mut *mut XlWorkbook, + ) -> c_int; + + pub fn xl_open_stream( + stream: *const XlStream, + format: i32, + options: *const XlOpenOptions, + out_handle: *mut *mut XlWorkbook, + ) -> c_int; + + pub fn xl_set_source_error(message: *const u8, len: i32); + pub fn xl_close(handle: *mut XlWorkbook) -> c_int; pub fn xl_sheet_count(handle: *mut XlWorkbook, out_count: *mut i32) -> c_int; diff --git a/rust/excelreader/src/options.rs b/rust/excelreader/src/options.rs index 4684b2d9..a14c58ae 100644 --- a/rust/excelreader/src/options.rs +++ b/rust/excelreader/src/options.rs @@ -38,6 +38,14 @@ pub struct OpenOptions { pub prefetch_decompression: Option, pub intern_strings: Option, + /// `open_source` only: bytes fetched per `read_at` call. `None` is 4 MiB; a negative value turns + /// the cache off. + pub source_block_size: Option, + /// `open_source` only: bytes of blocks kept in memory. `None` is 64 MiB. + pub source_cache_bytes: Option, + /// `open_reader` with XLSX, XLSB or XLS only: the most bytes the reader may hold. + pub max_buffered_bytes: Option, + /// Password for an encrypted OOXML workbook. `None` means "not encrypted, or fail with /// `Error::PasswordRequired`". pub password: Option, @@ -90,6 +98,13 @@ impl OpenOptions { prefetch_decompression: bool, /// XLS/XLSX/XLSB only. Library default is `false`. intern_strings: bool, + /// `open_source` only: bytes fetched per `read_at` call. Library default is 4 MiB; a negative + /// value turns the cache off. + source_block_size: i32, + /// `open_source` only: bytes of blocks kept in memory. Library default is 64 MiB. + source_cache_bytes: i64, + /// `open_reader` with XLSX, XLSB or XLS only: the most bytes the reader may hold. + max_buffered_bytes: i64, } /// Password for an encrypted OOXML workbook. Stored owned, since the C ABI receives a pointer @@ -130,6 +145,9 @@ impl OpenOptions { intern_strings: opt_state(self.intern_strings), password: std::ptr::null(), password_len: 0, + source_cache_bytes: opt_number(self.source_cache_bytes), + max_buffered_bytes: opt_number(self.max_buffered_bytes), + source_block_size: opt_number(self.source_block_size), }; if let Some(bytes) = password { raw.password = bytes.as_ptr(); @@ -367,4 +385,15 @@ mod tests { fn write_options_match_the_c_struct_size() { assert_eq!(std::mem::size_of::(), 32); } + + #[test] + fn source_layouts_match_the_c_structs() { + use crate::{XlOpenOptions, XlSource, XlStream}; + assert_eq!(std::mem::size_of::(), 112); + assert_eq!(std::mem::offset_of!(XlOpenOptions, source_cache_bytes), 88); + assert_eq!(std::mem::offset_of!(XlOpenOptions, max_buffered_bytes), 96); + assert_eq!(std::mem::offset_of!(XlOpenOptions, source_block_size), 104); + assert_eq!(std::mem::size_of::(), 40); + assert_eq!(std::mem::size_of::(), 32); + } } diff --git a/rust/excelreader/src/rows.rs b/rust/excelreader/src/rows.rs index 58a50217..c516de9a 100644 --- a/rust/excelreader/src/rows.rs +++ b/rust/excelreader/src/rows.rs @@ -229,7 +229,6 @@ pub struct RowCursor<'w> { } // SAFETY: the cursor owns its native enumerator and ABI v6 lets one thread at a time use it; -// every method that touches it takes `&mut self`. unsafe impl Send for RowCursor<'_> {} impl<'w> RowCursor<'w> { @@ -337,7 +336,6 @@ impl Drop for DecodedRows { } // SAFETY: plain native allocations, read only through &self, released by xl_free_rows, which any -// thread may call. unsafe impl Send for DecodedRows {} unsafe impl Sync for DecodedRows {} diff --git a/rust/excelreader/src/source.rs b/rust/excelreader/src/source.rs new file mode 100644 index 00000000..cee8f842 --- /dev/null +++ b/rust/excelreader/src/source.rs @@ -0,0 +1,94 @@ +//! Workbooks opened from bytes the caller serves: a positional [`Source`] or any [`Read`]. + +use crate::workbook::{check, check_abi_version, Workbook}; +use crate::{Error, OpenOptions, XlSource, XlStream, XlWorkbook, XL_INVALID_ARGUMENT}; +use std::any::Any; +use std::ffi::c_void; +use std::io::{self, Read}; +use std::panic::{catch_unwind, AssertUnwindSafe}; + +/// Random-access bytes for [`Workbook::open_source`]. The library calls `read_at` from several +/// threads at once, which is why the trait requires `Send + Sync`. +pub trait Source: Send + Sync + 'static { + /// Total size in bytes. Called once, while the workbook opens. + fn size(&self) -> io::Result; + + /// Fills `buf` with the bytes starting at `offset` and returns how many it wrote. Returning 0 + /// before the end is a failure. + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result; +} + +impl Workbook { + /// Opens a workbook over `source`, which the library owns from here on and drops once nothing + /// opened from the workbook needs it. Every sheet can be read in parallel. + pub fn open_source(source: S, format: i32, options: Option<&OpenOptions>) -> Result { + check_abi_version()?; + let size = source.size().map_err(|error| Error::from_status(XL_INVALID_ARGUMENT, error.to_string()))?; + let length = i64::try_from(size) + .map_err(|_| Error::from_status(XL_INVALID_ARGUMENT, "the source size does not fit in an i64".to_string()))?; + let raw = XlSource { + struct_size: std::mem::size_of::() as i32, + user_data: Box::into_raw(Box::new(source)).cast(), + length, + read_at: Some(read_at::), + release: Some(release::), + }; + let options = options.map(OpenOptions::to_raw); + let options_ptr = options + .as_ref() + .map_or(std::ptr::null(), crate::options::OpenOptionsRaw::as_ptr); + let mut handle: *mut XlWorkbook = std::ptr::null_mut(); + check(unsafe { crate::xl_open_source(&raw, format, options_ptr, &mut handle) })?; + Ok(Workbook::from_handle(handle)) + } + + /// Opens a workbook over `reader`, which cannot seek. A CSV is read as it arrives, and + /// again from the start until 16 MiB of it were read; an XLSX, XLSB or XLS is read whole first. The library owns `reader` from here on. + pub fn open_reader(reader: R, format: i32, options: Option<&OpenOptions>) -> Result { + check_abi_version()?; + let raw = XlStream { + struct_size: std::mem::size_of::() as i32, + user_data: Box::into_raw(Box::new(reader)).cast(), + read: Some(read::), + release: Some(release::), + }; + let options = options.map(OpenOptions::to_raw); + let options_ptr = options + .as_ref() + .map_or(std::ptr::null(), crate::options::OpenOptionsRaw::as_ptr); + let mut handle: *mut XlWorkbook = std::ptr::null_mut(); + check(unsafe { crate::xl_open_stream(&raw, format, options_ptr, &mut handle) })?; + Ok(Workbook::from_handle(handle)) + } +} + +unsafe extern "C" fn read_at(user_data: *mut c_void, offset: i64, buf: *mut u8, len: i64) -> i64 { + let source = &*user_data.cast::(); + let buf = std::slice::from_raw_parts_mut(buf, len as usize); + report(catch_unwind(AssertUnwindSafe(|| source.read_at(offset as u64, buf)))) +} + +unsafe extern "C" fn read(user_data: *mut c_void, buf: *mut u8, len: i64) -> i64 { + let reader = &mut *user_data.cast::(); + let buf = std::slice::from_raw_parts_mut(buf, len as usize); + report(catch_unwind(AssertUnwindSafe(|| loop { + match reader.read(buf) { + Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, + other => return other, + } + }))) +} + +unsafe extern "C" fn release(user_data: *mut c_void) { + let _ = catch_unwind(AssertUnwindSafe(|| drop(Box::from_raw(user_data.cast::())))); +} + +fn report(outcome: Result, Box>) -> i64 { + let message = match outcome { + Ok(Ok(count)) => return count as i64, + Ok(Err(error)) => error.to_string(), + Err(_) => "source panicked".to_string(), + }; + unsafe { crate::xl_set_source_error(message.as_ptr(), message.len() as i32) }; + -1 +} diff --git a/rust/excelreader/src/workbook.rs b/rust/excelreader/src/workbook.rs index 863fd498..14934735 100644 --- a/rust/excelreader/src/workbook.rs +++ b/rust/excelreader/src/workbook.rs @@ -135,11 +135,14 @@ pub struct Workbook { } // SAFETY: ABI v6 documents the workbook handle as usable from several threads at once. The only -// state behind it is immutable after open; each read owns its enumerator. unsafe impl Send for Workbook {} unsafe impl Sync for Workbook {} impl Workbook { + pub(crate) fn from_handle(handle: *mut XlWorkbook) -> Workbook { + Workbook { handle } + } + /// Opens `path`, sniffing the format and using every library default. /// /// Sniffing does NOT detect CSV - open one with [`open_with`](Self::open_with) and @@ -695,7 +698,6 @@ impl Drop for TableView { } // SAFETY: plain native allocations, read only through &self, released by xl_free_table, which any -// thread may call. T appears only as fn pointers, never as a stored value. unsafe impl Send for TableView {} unsafe impl Sync for TableView {} @@ -719,7 +721,6 @@ pub struct TypedChunks<'a, T: ExcelMapper> { } // SAFETY: a typed reader owns its native enumerator; ABI v6 lets it be used from any one thread -// at a time, which `&mut self` on `next` already guarantees. unsafe impl Send for TypedChunks<'_, T> {} impl Iterator for TypedChunks<'_, T> { diff --git a/rust/excelreader/src/writer_handle.rs b/rust/excelreader/src/writer_handle.rs index 0da1f6ae..1f8abdf7 100644 --- a/rust/excelreader/src/writer_handle.rs +++ b/rust/excelreader/src/writer_handle.rs @@ -32,7 +32,7 @@ pub struct WriterHandle { impl WriterHandle { /// Creates `path` (truncating it if it already exists), inferring the format from its - /// extension via [`format_from_path`](crate::writer::format_from_path). + /// extension via [`crate::writer::format_from_path`]. pub fn open(path: &str, options: Option<&WriteOptions>) -> Result { Self::open_with(path, format_from_path(path), options) } diff --git a/rust/excelreader/tests/source.rs b/rust/excelreader/tests/source.rs new file mode 100644 index 00000000..9167fb42 --- /dev/null +++ b/rust/excelreader/tests/source.rs @@ -0,0 +1,214 @@ +use excelreader::workbook::Workbook; +use excelreader::writer_handle::WriterHandle; +use excelreader::{OpenOptions, RowCursor, Source, XL_FORMAT_AUTO, XL_FORMAT_CSV, XL_FORMAT_XLSX}; +use std::io::{self, Read}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +const SHEETS: usize = 4; +const ROWS: usize = 300; + +fn workbook_bytes() -> Vec { + let mut writer = WriterHandle::open_memory(XL_FORMAT_XLSX, None).expect("open writer"); + for sheet in 0..SHEETS { + writer.start_sheet(&format!("sheet{sheet}")).expect("start sheet"); + for row in 0..ROWS { + writer.start_row().expect("start row"); + writer.write_str(Some(&format!("s{sheet}-r{row}"))).expect("write"); + writer.end_row().expect("end row"); + } + writer.end_sheet().expect("end sheet"); + } + writer.bytes().expect("bytes").to_vec() +} + +fn first_column(cursor: &mut RowCursor<'_>) -> Vec { + let mut values = Vec::new(); + while let Some(row) = cursor.next_row() { + let row = row.expect("a row"); + values.push(row.get(0).expect("a cell").as_str().expect("utf-8").to_string()); + } + values +} + +#[derive(Default, Clone)] +struct Counters { + reads: Arc, + drops: Arc, +} + +struct MemorySource { + bytes: Vec, + counters: Counters, +} + +impl Source for MemorySource { + fn size(&self) -> io::Result { + Ok(self.bytes.len() as u64) + } + + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result { + self.counters.reads.fetch_add(1, Ordering::SeqCst); + let start = (offset as usize).min(self.bytes.len()); + let count = buf.len().min(self.bytes.len() - start); + buf[..count].copy_from_slice(&self.bytes[start..start + count]); + Ok(count) + } +} + +impl Drop for MemorySource { + fn drop(&mut self) { + self.counters.drops.fetch_add(1, Ordering::SeqCst); + } +} + +struct FailingSource(Counters); + +impl Drop for FailingSource { + fn drop(&mut self) { + self.0.drops.fetch_add(1, Ordering::SeqCst); + } +} + +impl Source for FailingSource { + fn size(&self) -> io::Result { + Ok(1024) + } + + fn read_at(&self, _offset: u64, _buf: &mut [u8]) -> io::Result { + Err(io::Error::other("disk on fire")) + } +} + +struct PanickingSource(Counters); + +impl Drop for PanickingSource { + fn drop(&mut self) { + self.0.drops.fetch_add(1, Ordering::SeqCst); + } +} + +impl Source for PanickingSource { + fn size(&self) -> io::Result { + Ok(1024) + } + + fn read_at(&self, _offset: u64, _buf: &mut [u8]) -> io::Result { + panic!("boom"); + } +} + +struct OnlyRead(io::Cursor>, Counters); + +impl OnlyRead { + fn new(bytes: Vec, counters: &Counters) -> Self { + OnlyRead(io::Cursor::new(bytes), counters.clone()) + } +} + +impl Drop for OnlyRead { + fn drop(&mut self) { + self.1.drops.fetch_add(1, Ordering::SeqCst); + } +} + +impl Read for OnlyRead { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let limit = buf.len().min(7); + self.0.read(&mut buf[..limit]) + } +} + +#[test] +fn every_sheet_of_a_source_reads_in_parallel_like_memory() { + let bytes = workbook_bytes(); + let memory = Workbook::open_memory(&bytes, XL_FORMAT_AUTO, None).expect("open memory"); + let expected: Vec> = memory + .sheets() + .expect("sheets") + .iter() + .map(|sheet| first_column(&mut sheet.rows().expect("cursor"))) + .collect(); + + let counters = Counters::default(); + let workbook = Workbook::open_source(MemorySource { bytes, counters: counters.clone() }, XL_FORMAT_AUTO, None) + .expect("open source"); + let sheets = workbook.sheets().expect("sheets"); + let parallel: Vec> = std::thread::scope(|scope| { + let handles: Vec<_> = sheets + .iter() + .chain(sheets.iter()) + .map(|sheet| { + let sheet = *sheet; + scope.spawn(move || first_column(&mut sheet.rows().expect("cursor"))) + }) + .collect(); + handles.into_iter().map(|handle| handle.join().expect("no panic")).collect() + }); + for (index, rows) in parallel.iter().enumerate() { + assert_eq!(rows, &expected[index % SHEETS]); + } + + drop(workbook); + assert_eq!(counters.drops.load(Ordering::SeqCst), 1); +} + +#[test] +fn an_io_error_reaches_the_caller_with_its_message() { + let counters = Counters::default(); + let error = Workbook::open_source(FailingSource(counters.clone()), XL_FORMAT_XLSX, None).expect_err("must fail"); + assert!(error.message().contains("disk on fire"), "{}", error.message()); + assert_eq!(counters.drops.load(Ordering::SeqCst), 1); +} + +#[test] +fn a_panicking_source_becomes_an_error() { + let counters = Counters::default(); + let error = Workbook::open_source(PanickingSource(counters.clone()), XL_FORMAT_XLSX, None).expect_err("must fail"); + assert!(error.message().contains("source panicked"), "{}", error.message()); + assert_eq!(counters.drops.load(Ordering::SeqCst), 1); +} + +#[test] +fn the_default_cache_fetches_a_small_workbook_once() { + let bytes = workbook_bytes(); + let cached = Counters::default(); + { + let workbook = Workbook::open_source(MemorySource { bytes: bytes.clone(), counters: cached.clone() }, XL_FORMAT_XLSX, None) + .expect("open"); + first_column(&mut workbook.sheet(0).expect("sheet").rows().expect("cursor")); + } + assert_eq!(cached.reads.load(Ordering::SeqCst), 1); + + let uncached = Counters::default(); + let options = OpenOptions::new().source_block_size(-1); + { + let workbook = Workbook::open_source(MemorySource { bytes, counters: uncached.clone() }, XL_FORMAT_XLSX, Some(&options)) + .expect("open"); + first_column(&mut workbook.sheet(0).expect("sheet").rows().expect("cursor")); + } + assert!(uncached.reads.load(Ordering::SeqCst) > 1); +} + +#[test] +fn a_reader_serves_a_csv_and_an_xlsx() { + let csv_counters = Counters::default(); + let csv = OnlyRead::new(b"name,qty +widget,7 +gadget,9 +".to_vec(), &csv_counters); + let workbook = Workbook::open_reader(csv, XL_FORMAT_CSV, None).expect("open csv"); + let names = first_column(&mut workbook.sheet(0).expect("sheet").rows().expect("cursor")); + assert_eq!(names, vec!["name", "widget", "gadget"]); + assert_eq!(csv_counters.drops.load(Ordering::SeqCst), 0, "the CSV is read as it arrives"); + drop(workbook); + assert_eq!(csv_counters.drops.load(Ordering::SeqCst), 1); + + let xlsx_counters = Counters::default(); + let xlsx = OnlyRead::new(workbook_bytes(), &xlsx_counters); + let workbook = Workbook::open_reader(xlsx, XL_FORMAT_AUTO, None).expect("open xlsx"); + assert_eq!(xlsx_counters.drops.load(Ordering::SeqCst), 1, "an XLSX is read whole before open returns"); + let rows = first_column(&mut workbook.sheet(3).expect("sheet").rows().expect("cursor")); + assert_eq!(rows.len(), ROWS); + assert_eq!(rows[0], "s3-r0"); +} diff --git a/src/ExcelReader.Arrow/ArrowConversionExtensions.cs b/src/ExcelReader.Arrow/ArrowConversionExtensions.cs index a06e89be..25f7cdd3 100644 --- a/src/ExcelReader.Arrow/ArrowConversionExtensions.cs +++ b/src/ExcelReader.Arrow/ArrowConversionExtensions.cs @@ -9,26 +9,24 @@ public static class ArrowConversionExtensions { /// Converts one sheet into an Arrow . /// The sheet to convert. - /// The column schema to use; inferred from the sheet with when . - /// - /// 1-based header row, reused both for schema inference (when is - /// ) and to skip the header line during the data pass. 0 means no header row. - /// + /// The schema and header row; when . /// A record batch with one column per schema entry and one row per data row. /// is . /// /// A cell failed to convert to its column's declared type on a non-nullable column. /// - /// The sheet has fewer rows than . - /// names an unrecognized . - public static RecordBatch ToArrowRecordBatch(this IExcelSheet sheet, ExcelColumnSchema[]? schema = null, int headerRow = 1) + /// The sheet has fewer rows than . + /// names an unrecognized . + public static RecordBatch ToArrowRecordBatch(this IExcelSheet sheet, ArrowReadOptions? options = null) { ArgumentNullException.ThrowIfNull(sheet); - schema ??= Excel.InferSchema(sheet, headerRow); + options ??= ArrowReadOptions.Default; + int headerRow = options.HeaderRow; + IReadOnlyList schema = options.Schema ?? Excel.InferSchema(sheet, new SchemaInferenceOptions { HeaderRow = headerRow }); bool isDate1904 = sheet.IsDate1904; using IExcelRowEnumerator rows = sheet.GetEnumerator(); - ColumnAppender[] appenders = new ColumnAppender[schema.Length]; - for (int i = 0; i < schema.Length; i++) + ColumnAppender[] appenders = new ColumnAppender[schema.Count]; + for (int i = 0; i < schema.Count; i++) { appenders[i] = ColumnAppender.Create(schema[i]); } diff --git a/src/ExcelReader.Arrow/ArrowReadOptions.cs b/src/ExcelReader.Arrow/ArrowReadOptions.cs new file mode 100644 index 00000000..60606bc4 --- /dev/null +++ b/src/ExcelReader.Arrow/ArrowReadOptions.cs @@ -0,0 +1,24 @@ +using ExcelReader.Core.Reader.Schema; + +namespace ExcelReader.Arrow +{ + /// Options for converting a sheet with . + public sealed class ArrowReadOptions + { + /// Gets the default options: header on row 1, schema inferred from the sheet. + public static ArrowReadOptions Default { get; } = new(); + + /// + /// Gets the column schema to use. , the default, infers it from the sheet with + /// using . + /// + public IReadOnlyList? Schema { get; init; } + + /// + /// Gets the 1-based header row, used both for schema inference (when is + /// ) and to skip the header line during the data pass. 0 means no header + /// row. Defaults to 1. + /// + public int HeaderRow { get; init; } = 1; + } +} diff --git a/src/ExcelReader.Arrow/ArrowWriteExtensions.cs b/src/ExcelReader.Arrow/ArrowWriteExtensions.cs index 0b74138b..06266f89 100644 --- a/src/ExcelReader.Arrow/ArrowWriteExtensions.cs +++ b/src/ExcelReader.Arrow/ArrowWriteExtensions.cs @@ -11,7 +11,7 @@ namespace ExcelReader.Arrow { /// /// Writes an Apache Arrow to a workbook as one sheet — the write-side - /// mirror of . + /// mirror of . /// public static class ArrowWriteExtensions { @@ -20,7 +20,7 @@ public static class ArrowWriteExtensions /// /// Writes to as one XLSX sheet, in a single - /// call: adds the sheet, writes the header row (unless is false) + /// call: adds the sheet, writes the header row (unless is false) /// and every data row, then ends the sheet and the workbook. /// /// A freshly created, not-yet-started workbook writer. @@ -29,78 +29,78 @@ public static class ArrowWriteExtensions /// width (signed and unsigned), half float, float, double, decimal128, decimal256, boolean, /// date32, date64, time32, time64, timestamp and null. /// - /// The sheet's name. - /// Whether to write a header row of column names first. + /// The sheet name and whether to write a header row; when . /// or is . /// A column's Arrow type is not one of the supported types. - public static void WriteRecordBatch(this XlsxWorkbookWriter workbook, RecordBatch batch, string sheetName = "Sheet1", bool writeHeader = true) + public static void WriteRecordBatch(this XlsxWorkbookWriter workbook, RecordBatch batch, ArrowWriteOptions? options = null) { - WriteRecordBatchCore(workbook, batch, sheetName, writeHeader); + WriteRecordBatchCore(workbook, batch, options); } - /// Same as , for an XLSB workbook. + /// Same as , for an XLSB workbook. /// or is . /// A column's Arrow type is not one of the supported types. - public static void WriteRecordBatch(this XlsbWorkbookWriter workbook, RecordBatch batch, string sheetName = "Sheet1", bool writeHeader = true) + public static void WriteRecordBatch(this XlsbWorkbookWriter workbook, RecordBatch batch, ArrowWriteOptions? options = null) { - WriteRecordBatchCore(workbook, batch, sheetName, writeHeader); + WriteRecordBatchCore(workbook, batch, options); } - /// Same as , for a legacy XLS workbook. + /// Same as , for a legacy XLS workbook. /// or is . /// A column's Arrow type is not one of the supported types. - public static void WriteRecordBatch(this XlsWorkbookWriter workbook, RecordBatch batch, string sheetName = "Sheet1", bool writeHeader = true) + public static void WriteRecordBatch(this XlsWorkbookWriter workbook, RecordBatch batch, ArrowWriteOptions? options = null) { - WriteRecordBatchCore(workbook, batch, sheetName, writeHeader); + WriteRecordBatchCore(workbook, batch, options); } /// /// Writes to as CSV's single unnamed sheet, - /// in a single call. + /// in a single call. is ignored. /// /// or is . /// A column's Arrow type is not one of the supported types. - public static void WriteRecordBatch(this CsvWorkbookWriter workbook, RecordBatch batch, bool writeHeader = true) + public static void WriteRecordBatch(this CsvWorkbookWriter workbook, RecordBatch batch, ArrowWriteOptions? options = null) { - WriteRecordBatchCore(workbook, batch, sheetName: "Sheet1", writeHeader); + WriteRecordBatchCore(workbook, batch, options); } - /// Asynchronous counterpart to . - public static ValueTask WriteRecordBatchAsync(this XlsxWorkbookWriter workbook, RecordBatch batch, string sheetName = "Sheet1", bool writeHeader = true, CancellationToken ct = default) + /// Asynchronous counterpart to . + public static ValueTask WriteRecordBatchAsync(this XlsxWorkbookWriter workbook, RecordBatch batch, ArrowWriteOptions? options = null, CancellationToken ct = default) { - return WriteRecordBatchCoreAsync(workbook, batch, sheetName, writeHeader, ct); + return WriteRecordBatchCoreAsync(workbook, batch, options, ct); } - /// Asynchronous counterpart to . - public static ValueTask WriteRecordBatchAsync(this XlsbWorkbookWriter workbook, RecordBatch batch, string sheetName = "Sheet1", bool writeHeader = true, CancellationToken ct = default) + /// Asynchronous counterpart to . + public static ValueTask WriteRecordBatchAsync(this XlsbWorkbookWriter workbook, RecordBatch batch, ArrowWriteOptions? options = null, CancellationToken ct = default) { - return WriteRecordBatchCoreAsync(workbook, batch, sheetName, writeHeader, ct); + return WriteRecordBatchCoreAsync(workbook, batch, options, ct); } - /// Asynchronous counterpart to . - public static ValueTask WriteRecordBatchAsync(this XlsWorkbookWriter workbook, RecordBatch batch, string sheetName = "Sheet1", bool writeHeader = true, CancellationToken ct = default) + /// Asynchronous counterpart to . + public static ValueTask WriteRecordBatchAsync(this XlsWorkbookWriter workbook, RecordBatch batch, ArrowWriteOptions? options = null, CancellationToken ct = default) { - return WriteRecordBatchCoreAsync(workbook, batch, sheetName, writeHeader, ct); + return WriteRecordBatchCoreAsync(workbook, batch, options, ct); } - /// Asynchronous counterpart to . - public static ValueTask WriteRecordBatchAsync(this CsvWorkbookWriter workbook, RecordBatch batch, bool writeHeader = true, CancellationToken ct = default) + /// Asynchronous counterpart to . + public static ValueTask WriteRecordBatchAsync(this CsvWorkbookWriter workbook, RecordBatch batch, ArrowWriteOptions? options = null, CancellationToken ct = default) { - return WriteRecordBatchCoreAsync(workbook, batch, sheetName: "Sheet1", writeHeader, ct); + return WriteRecordBatchCoreAsync(workbook, batch, options, ct); } - private static void WriteRecordBatchCore(IWorkbookWriter workbook, RecordBatch batch, string sheetName, bool writeHeader) + private static void WriteRecordBatchCore(IWorkbookWriter workbook, RecordBatch batch, ArrowWriteOptions? options) where TSheet : ISheetWriter where TRow : IRowWriter { ArgumentNullException.ThrowIfNull(workbook); ArgumentNullException.ThrowIfNull(batch); + options ??= ArrowWriteOptions.Default; IReadOnlyList fields = batch.Schema.FieldsList; - TSheet sheet = workbook.AddSheet(sheetName); + TSheet sheet = workbook.AddSheet(options.SheetName); ApplyTemporalStyles(workbook, sheet, fields); - if (writeHeader) + if (options.WriteHeader) { using TRow header = sheet.StartRow(); foreach (Field field in fields) @@ -122,18 +122,19 @@ private static void WriteRecordBatchCore(IWorkbookWriter w workbook.End(); } - private static async ValueTask WriteRecordBatchCoreAsync(IWorkbookWriter workbook, RecordBatch batch, string sheetName, bool writeHeader, CancellationToken ct) + private static async ValueTask WriteRecordBatchCoreAsync(IWorkbookWriter workbook, RecordBatch batch, ArrowWriteOptions? options, CancellationToken ct) where TSheet : ISheetWriter where TRow : IRowWriter { ArgumentNullException.ThrowIfNull(workbook); ArgumentNullException.ThrowIfNull(batch); + options ??= ArrowWriteOptions.Default; IReadOnlyList fields = batch.Schema.FieldsList; - TSheet sheet = workbook.AddSheet(sheetName); + TSheet sheet = workbook.AddSheet(options.SheetName); ApplyTemporalStyles(workbook, sheet, fields); - if (writeHeader) + if (options.WriteHeader) { TRow header = await sheet.StartRowAsync(ct).ConfigureAwait(false); foreach (Field field in fields) diff --git a/src/ExcelReader.Arrow/ArrowWriteOptions.cs b/src/ExcelReader.Arrow/ArrowWriteOptions.cs new file mode 100644 index 00000000..e0d97653 --- /dev/null +++ b/src/ExcelReader.Arrow/ArrowWriteOptions.cs @@ -0,0 +1,15 @@ +namespace ExcelReader.Arrow +{ + /// Options for writing a record batch with . + public sealed class ArrowWriteOptions + { + /// Gets the default options: a sheet named Sheet1 with a header row. + public static ArrowWriteOptions Default { get; } = new(); + + /// Gets the name of the sheet the batch is written to. Defaults to Sheet1. + public string SheetName { get; init; } = "Sheet1"; + + /// Gets whether the first row holds the batch's field names. Defaults to . + public bool WriteHeader { get; init; } = true; + } +} diff --git a/src/ExcelReader.Arrow/ColumnAppender.cs b/src/ExcelReader.Arrow/ColumnAppender.cs index 4ab6ae78..7b5e9376 100644 --- a/src/ExcelReader.Arrow/ColumnAppender.cs +++ b/src/ExcelReader.Arrow/ColumnAppender.cs @@ -175,7 +175,7 @@ internal override Field Field internal override void Append(in Cell cell, bool isDate1904) { - if (ExcelCellReaders.Bool(in cell, isDate1904, CultureInfo.InvariantCulture, out bool value)) + if (ExcelCellReaders.Bool(in cell, new ExcelRowContext { IsDate1904 = isDate1904 }, out bool value)) { _builder.Append(value); } @@ -210,7 +210,7 @@ internal override Field Field internal override void Append(in Cell cell, bool isDate1904) { - if (ExcelCellReaders.DateOnlyAuto(in cell, isDate1904, CultureInfo.InvariantCulture, out DateOnly value)) + if (ExcelCellReaders.DateOnlyAuto(in cell, new ExcelRowContext { IsDate1904 = isDate1904 }, out DateOnly value)) { _builder.Append(value); } @@ -247,7 +247,7 @@ internal override Field Field internal override void Append(in Cell cell, bool isDate1904) { - if (ExcelCellReaders.TimeOnlyAuto(in cell, isDate1904, CultureInfo.InvariantCulture, out TimeOnly value)) + if (ExcelCellReaders.TimeOnlyAuto(in cell, new ExcelRowContext { IsDate1904 = isDate1904 }, out TimeOnly value)) { _builder.Append(value.ToTimeSpan().Ticks / 10); } @@ -284,7 +284,7 @@ internal override Field Field internal override void Append(in Cell cell, bool isDate1904) { - if (ExcelCellReaders.DateTimeAuto(in cell, isDate1904, CultureInfo.InvariantCulture, out DateTime value)) + if (ExcelCellReaders.DateTimeAuto(in cell, new ExcelRowContext { IsDate1904 = isDate1904 }, out DateTime value)) { DateTime normalized = value.Kind == DateTimeKind.Unspecified ? value diff --git a/src/ExcelReader.Arrow/ExcelReader.Arrow.csproj b/src/ExcelReader.Arrow/ExcelReader.Arrow.csproj index 6b2a5232..2e0e098e 100644 --- a/src/ExcelReader.Arrow/ExcelReader.Arrow.csproj +++ b/src/ExcelReader.Arrow/ExcelReader.Arrow.csproj @@ -14,6 +14,7 @@ excel;arrow;apache-arrow;recordbatch;interop MIT README.md + icon.png https://github.com/GabrielMarquezMatte/ExcelReader https://github.com/GabrielMarquezMatte/ExcelReader git @@ -32,6 +33,7 @@ + diff --git a/src/ExcelReader.Arrow/PublicAPI/PublicAPI.Shipped.txt b/src/ExcelReader.Arrow/PublicAPI/PublicAPI.Shipped.txt index 6a1803ef..a10d55c4 100644 --- a/src/ExcelReader.Arrow/PublicAPI/PublicAPI.Shipped.txt +++ b/src/ExcelReader.Arrow/PublicAPI/PublicAPI.Shipped.txt @@ -1,12 +1,3 @@ #nullable enable ExcelReader.Arrow.ArrowConversionExtensions ExcelReader.Arrow.ArrowWriteExtensions -static ExcelReader.Arrow.ArrowConversionExtensions.ToArrowRecordBatch(this ExcelReader.Core.Reader.IExcelRowReader! reader, ExcelReader.Core.Reader.Schema.ExcelColumnSchema[]? schema = null, int headerRow = 1) -> Apache.Arrow.RecordBatch! -static ExcelReader.Arrow.ArrowWriteExtensions.WriteRecordBatch(this ExcelReader.Core.Writer.Csv.CsvWorkbookWriter! workbook, Apache.Arrow.RecordBatch! batch, bool writeHeader = true) -> void -static ExcelReader.Arrow.ArrowWriteExtensions.WriteRecordBatch(this ExcelReader.Core.Writer.Xls.XlsWorkbookWriter! workbook, Apache.Arrow.RecordBatch! batch, string! sheetName = "Sheet1", bool writeHeader = true) -> void -static ExcelReader.Arrow.ArrowWriteExtensions.WriteRecordBatch(this ExcelReader.Core.Writer.Xlsb.XlsbWorkbookWriter! workbook, Apache.Arrow.RecordBatch! batch, string! sheetName = "Sheet1", bool writeHeader = true) -> void -static ExcelReader.Arrow.ArrowWriteExtensions.WriteRecordBatch(this ExcelReader.Core.Writer.Xlsx.XlsxWorkbookWriter! workbook, Apache.Arrow.RecordBatch! batch, string! sheetName = "Sheet1", bool writeHeader = true) -> void -static ExcelReader.Arrow.ArrowWriteExtensions.WriteRecordBatchAsync(this ExcelReader.Core.Writer.Csv.CsvWorkbookWriter! workbook, Apache.Arrow.RecordBatch! batch, bool writeHeader = true, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask -static ExcelReader.Arrow.ArrowWriteExtensions.WriteRecordBatchAsync(this ExcelReader.Core.Writer.Xls.XlsWorkbookWriter! workbook, Apache.Arrow.RecordBatch! batch, string! sheetName = "Sheet1", bool writeHeader = true, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask -static ExcelReader.Arrow.ArrowWriteExtensions.WriteRecordBatchAsync(this ExcelReader.Core.Writer.Xlsb.XlsbWorkbookWriter! workbook, Apache.Arrow.RecordBatch! batch, string! sheetName = "Sheet1", bool writeHeader = true, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask -static ExcelReader.Arrow.ArrowWriteExtensions.WriteRecordBatchAsync(this ExcelReader.Core.Writer.Xlsx.XlsxWorkbookWriter! workbook, Apache.Arrow.RecordBatch! batch, string! sheetName = "Sheet1", bool writeHeader = true, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask diff --git a/src/ExcelReader.Arrow/PublicAPI/PublicAPI.Unshipped.txt b/src/ExcelReader.Arrow/PublicAPI/PublicAPI.Unshipped.txt index 38a178ff..854fae10 100644 --- a/src/ExcelReader.Arrow/PublicAPI/PublicAPI.Unshipped.txt +++ b/src/ExcelReader.Arrow/PublicAPI/PublicAPI.Unshipped.txt @@ -1,3 +1,24 @@ #nullable enable -static ExcelReader.Arrow.ArrowConversionExtensions.ToArrowRecordBatch(this ExcelReader.Core.Reader.IExcelSheet! sheet, ExcelReader.Core.Reader.Schema.ExcelColumnSchema[]? schema = null, int headerRow = 1) -> Apache.Arrow.RecordBatch! -*REMOVED*static ExcelReader.Arrow.ArrowConversionExtensions.ToArrowRecordBatch(this ExcelReader.Core.Reader.IExcelRowReader! reader, ExcelReader.Core.Reader.Schema.ExcelColumnSchema[]? schema = null, int headerRow = 1) -> Apache.Arrow.RecordBatch! +ExcelReader.Arrow.ArrowReadOptions +ExcelReader.Arrow.ArrowReadOptions.ArrowReadOptions() -> void +ExcelReader.Arrow.ArrowReadOptions.HeaderRow.get -> int +ExcelReader.Arrow.ArrowReadOptions.HeaderRow.init -> void +ExcelReader.Arrow.ArrowReadOptions.Schema.get -> System.Collections.Generic.IReadOnlyList? +ExcelReader.Arrow.ArrowReadOptions.Schema.init -> void +ExcelReader.Arrow.ArrowWriteOptions +ExcelReader.Arrow.ArrowWriteOptions.ArrowWriteOptions() -> void +ExcelReader.Arrow.ArrowWriteOptions.SheetName.get -> string! +ExcelReader.Arrow.ArrowWriteOptions.SheetName.init -> void +ExcelReader.Arrow.ArrowWriteOptions.WriteHeader.get -> bool +ExcelReader.Arrow.ArrowWriteOptions.WriteHeader.init -> void +static ExcelReader.Arrow.ArrowConversionExtensions.ToArrowRecordBatch(this ExcelReader.Core.Reader.IExcelSheet! sheet, ExcelReader.Arrow.ArrowReadOptions? options = null) -> Apache.Arrow.RecordBatch! +static ExcelReader.Arrow.ArrowReadOptions.Default.get -> ExcelReader.Arrow.ArrowReadOptions! +static ExcelReader.Arrow.ArrowWriteExtensions.WriteRecordBatch(this ExcelReader.Core.Writer.Csv.CsvWorkbookWriter! workbook, Apache.Arrow.RecordBatch! batch, ExcelReader.Arrow.ArrowWriteOptions? options = null) -> void +static ExcelReader.Arrow.ArrowWriteExtensions.WriteRecordBatch(this ExcelReader.Core.Writer.Xls.XlsWorkbookWriter! workbook, Apache.Arrow.RecordBatch! batch, ExcelReader.Arrow.ArrowWriteOptions? options = null) -> void +static ExcelReader.Arrow.ArrowWriteExtensions.WriteRecordBatch(this ExcelReader.Core.Writer.Xlsb.XlsbWorkbookWriter! workbook, Apache.Arrow.RecordBatch! batch, ExcelReader.Arrow.ArrowWriteOptions? options = null) -> void +static ExcelReader.Arrow.ArrowWriteExtensions.WriteRecordBatch(this ExcelReader.Core.Writer.Xlsx.XlsxWorkbookWriter! workbook, Apache.Arrow.RecordBatch! batch, ExcelReader.Arrow.ArrowWriteOptions? options = null) -> void +static ExcelReader.Arrow.ArrowWriteExtensions.WriteRecordBatchAsync(this ExcelReader.Core.Writer.Csv.CsvWorkbookWriter! workbook, Apache.Arrow.RecordBatch! batch, ExcelReader.Arrow.ArrowWriteOptions? options = null, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask +static ExcelReader.Arrow.ArrowWriteExtensions.WriteRecordBatchAsync(this ExcelReader.Core.Writer.Xls.XlsWorkbookWriter! workbook, Apache.Arrow.RecordBatch! batch, ExcelReader.Arrow.ArrowWriteOptions? options = null, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask +static ExcelReader.Arrow.ArrowWriteExtensions.WriteRecordBatchAsync(this ExcelReader.Core.Writer.Xlsb.XlsbWorkbookWriter! workbook, Apache.Arrow.RecordBatch! batch, ExcelReader.Arrow.ArrowWriteOptions? options = null, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask +static ExcelReader.Arrow.ArrowWriteExtensions.WriteRecordBatchAsync(this ExcelReader.Core.Writer.Xlsx.XlsxWorkbookWriter! workbook, Apache.Arrow.RecordBatch! batch, ExcelReader.Arrow.ArrowWriteOptions? options = null, System.Threading.CancellationToken ct = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask +static ExcelReader.Arrow.ArrowWriteOptions.Default.get -> ExcelReader.Arrow.ArrowWriteOptions! diff --git a/src/ExcelReader.Cli/CliCommands.cs b/src/ExcelReader.Cli/CliCommands.cs index 0105637e..8f1833d8 100644 --- a/src/ExcelReader.Cli/CliCommands.cs +++ b/src/ExcelReader.Cli/CliCommands.cs @@ -120,7 +120,7 @@ private static void WriteCsv(IExcelSheet sheet, Stream target, bool leaveOpen, c private static void WriteXlsx(IExcelSheet sheet, Stream target, bool leaveOpen, Action? onProgress) { - using XlsxWorkbookWriter workbook = XlsxWorkbookWriter.Create(target, leaveOpen); + using XlsxWorkbookWriter workbook = XlsxWorkbookWriter.Create(target, leaveOpen, new XlsxWriterOptions { Date1904 = sheet.IsDate1904 }); WriteRows(workbook, sheet, onProgress); } @@ -132,7 +132,7 @@ private static void WriteXlsb(IExcelSheet sheet, Stream target, bool leaveOpen, private static void WriteXls(IExcelSheet sheet, Stream target, bool leaveOpen, Action? onProgress) { - using XlsWorkbookWriter workbook = XlsWorkbookWriter.Create(target, leaveOpen, date1904: sheet.IsDate1904); + using XlsWorkbookWriter workbook = XlsWorkbookWriter.Create(target, leaveOpen, new XlsWriterOptions { Date1904 = sheet.IsDate1904 }); WriteRows(workbook, sheet, onProgress); } @@ -170,7 +170,7 @@ private static void WriteRows(TWorkbook workbook, IExce } break; case CellType.Date: - var dateValue = cellValue.TryGetDateTime(out var date) ? date : throw new InvalidOperationException($"cell {cell.ColumnIndex} is a date but TryGetDateTime failed to parse it."); + var dateValue = cellValue.TryGetDateTime(sheet.IsDate1904, out var date) ? date : throw new InvalidOperationException($"cell {cell.ColumnIndex} is a date but TryGetDateTime failed to parse it."); row.Write(dateValue); break; default: @@ -195,7 +195,7 @@ internal static int Schema(string path, string? sheet, int headerRow, int sample { using IExcelWorkbook workbook = Open(path, sheet, out IExcelSheet selected, password, inputDelimiter); - foreach (ExcelColumnSchema column in Excel.InferSchema(selected, headerRow, sampleSize)) + foreach (ExcelColumnSchema column in Excel.InferSchema(selected, new SchemaInferenceOptions { HeaderRow = headerRow, SampleSize = sampleSize })) { stdout.Write(column.Index.ToString(CultureInfo.InvariantCulture)); stdout.Write('\t'); diff --git a/src/ExcelReader.Core/ExcelReader.Core.csproj b/src/ExcelReader.Core/ExcelReader.Core.csproj index 216104fc..3bf3c80e 100644 --- a/src/ExcelReader.Core/ExcelReader.Core.csproj +++ b/src/ExcelReader.Core/ExcelReader.Core.csproj @@ -18,6 +18,7 @@ excel;xlsx;xlsb;xls;csv;reader;writer;parser;spreadsheet;streaming;performance;low-allocation MIT README.md + icon.png https://github.com/GabrielMarquezMatte/ExcelReader https://github.com/GabrielMarquezMatte/ExcelReader git @@ -36,6 +37,8 @@ + + @@ -47,6 +50,8 @@ + + z")] + public async Task ARowLikeTagInsideARowMatchesSequential(string poison) + { + using MemoryStream ms = WorkbookBuilder.Build(Rows(120, r => r % 10 == 0 ? poison : "n")); + using XlsxWorkbook workbook = Excel.FromXlsx(ms); + List expected = Sequential(workbook, 0); + + foreach (int chunkBytes in new[] { 64, 90, 150, 400 }) + { + Assert.Equal(expected, await PipelineAsync(workbook, 0, 4, chunkBytes)); + Assert.Equal(expected[1..], await PipelineAsync(workbook, 1, 4, chunkBytes)); + } + } + + [Fact] + public async Task RowsAfterTheEndOfSheetDataAreIgnored() + { + string rows = Rows(60) + "" + Rows(60) + ""; + using MemoryStream ms = WorkbookBuilder.Build(rows); + using XlsxWorkbook workbook = Excel.FromXlsx(ms); + List expected = Sequential(workbook, 0); + + Assert.Equal(60, expected.Count); + Assert.Equal(expected, await PipelineAsync(workbook, 0, 4, 80)); + } + + [Fact] + public async Task AnAccumulateExceptionSurfaces() + { + using MemoryStream ms = WorkbookBuilder.Build(Rows(300)); + using XlsxWorkbook workbook = Excel.FromXlsx(ms); + var failing = new RowAggregation + { + Seed = static () => 0, + Accumulate = static (ref int count, Row row) => + { + if (row[1].TryGetDouble(out double value) && value == 150) + { + throw new InvalidOperationException("boom"); + } + count++; + }, + Combine = static (left, right) => left + right, + }; + SheetPipelineSource source = SheetPipelineSource.Create(workbook.FirstSheet)!; + + InvalidOperationException ex = await Assert.ThrowsAsync( + () => SheetPipelineRunner.AggregateAsync(source, failing, null, 0, 4, 128, TestContext.Current.CancellationToken)); + + Assert.Equal("boom", ex.Message); + } + + [Fact] + public async Task CancellationStopsTheRead() + { + using MemoryStream ms = WorkbookBuilder.Build(Rows(2000)); + using XlsxWorkbook workbook = Excel.FromXlsx(ms); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(TestContext.Current.CancellationToken); + int seen = 0; + var cancelling = new RowAggregation + { + Seed = static () => 0, + Accumulate = (ref int count, Row _) => + { + count++; + if (Interlocked.Increment(ref seen) == 5) + { + cts.Cancel(); + } + }, + Combine = static (left, right) => left + right, + }; + SheetPipelineSource source = SheetPipelineSource.Create(workbook.FirstSheet)!; + + await Assert.ThrowsAnyAsync( + () => SheetPipelineRunner.AggregateAsync(source, cancelling, null, 0, 4, 64, cts.Token)); + } + } +} diff --git a/tests/ExcelReader.Tests/Parser/UnifiedParserTests.cs b/tests/ExcelReader.Tests/Parser/UnifiedParserTests.cs index 40265324..ba7f6f38 100644 --- a/tests/ExcelReader.Tests/Parser/UnifiedParserTests.cs +++ b/tests/ExcelReader.Tests/Parser/UnifiedParserTests.cs @@ -154,14 +154,14 @@ public void RefStructModelParsesThroughExcelFluentParser() { using CsvReader reader = Excel.FromCsv(Csv); var parser = ExcelParser.Build(static builder => builder - .PropertyRaw(["Name"], static (ref FluentSale m, in Cell c, bool _, IFormatProvider _) => + .PropertyRaw(["Name"], static (ref FluentSale m, in Cell c, in ExcelRowContext _) => { m.Name = c.Value; return true; }) - .PropertyRaw(["Id"], static (ref FluentSale m, in Cell c, bool _, IFormatProvider p) => + .PropertyRaw(["Id"], static (ref FluentSale m, in Cell c, in ExcelRowContext x) => { - bool ok = c.TryParse(p, out int id); + bool ok = c.TryParse(x.FormatProvider, out int id); m.Id = id; return ok; })); diff --git a/tests/ExcelReader.Tests/Reader/CellVariantTests.cs b/tests/ExcelReader.Tests/Reader/CellVariantTests.cs index 08ee5612..2bc0f781 100644 --- a/tests/ExcelReader.Tests/Reader/CellVariantTests.cs +++ b/tests/ExcelReader.Tests/Reader/CellVariantTests.cs @@ -31,14 +31,14 @@ public void ErrorCellHasErrorType() } [Fact] - public void FormulaCellHasFormulaType() + public void AFormulaTextResultIsAString() { using var ms = WorkbookBuilder.Build( """Hello"""); using var reader = Excel.FromXlsx(ms); using var e = reader.FirstSheet.GetEnumerator(); Assert.True(e.MoveNext()); - Assert.Equal(CellType.Formula, e.Current[0].Type); + Assert.Equal(CellType.ExcelString, e.Current[0].Type); Assert.Equal("Hello", e.Current[0].GetString()); } @@ -167,7 +167,7 @@ public async Task TryGetDateTimeOnInlineStringReturnsFalse() await using var reader = await Excel.FromXlsxAsync(ms, ct: TestContext.Current.CancellationToken); await using var e = reader.FirstSheet.GetEnumerator(); Assert.True(await e.MoveNextAsync()); - Assert.False(e.Current[0].TryGetDateTime(out _)); + Assert.False(e.Current[0].TryGetDateTime(isDate1904: false, out _)); } [Fact] @@ -177,7 +177,7 @@ public async Task TryGetDateTimeOutOfRangeSerialReturnsFalse() await using var reader = await Excel.FromXlsxAsync(ms, ct: TestContext.Current.CancellationToken); await using var e = reader.FirstSheet.GetEnumerator(); Assert.True(await e.MoveNextAsync()); - Assert.False(e.Current[0].TryGetDateTime(out _)); + Assert.False(e.Current[0].TryGetDateTime(isDate1904: false, out _)); } [Fact] @@ -278,9 +278,9 @@ public void Excel1900SerialsOneToSixtyMapToExcelCalendar() using var e = reader.FirstSheet.GetEnumerator(); Assert.True(e.MoveNext()); - Assert.True(e.Current[0].TryGetDateTime(out DateTime first)); - Assert.True(e.Current[1].TryGetDateTime(out DateTime leapBoundary)); - Assert.True(e.Current[2].TryGetDateTime(out DateTime phantomLeapDay)); + Assert.True(e.Current[0].TryGetDateTime(isDate1904: false, out DateTime first)); + Assert.True(e.Current[1].TryGetDateTime(isDate1904: false, out DateTime leapBoundary)); + Assert.True(e.Current[2].TryGetDateTime(isDate1904: false, out DateTime phantomLeapDay)); Assert.Equal(new DateTime(1900, 1, 1), first); Assert.Equal(new DateTime(1900, 2, 28), leapBoundary); Assert.Equal(new DateTime(1900, 2, 28), phantomLeapDay); diff --git a/tests/ExcelReader.Tests/Reader/ChunkEnumeratorTests.cs b/tests/ExcelReader.Tests/Reader/ChunkEnumeratorTests.cs new file mode 100644 index 00000000..57b4f603 --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/ChunkEnumeratorTests.cs @@ -0,0 +1,182 @@ +using System.Globalization; +using System.Text; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Xlsb; +using ExcelReader.Core.Reader.Xlsx; +using ExcelReader.Core.Writer.Xlsb; + +namespace ExcelReader.Tests.Reader +{ + public class ChunkEnumeratorTests + { + private static string Render(Row row) + { + var sb = new StringBuilder(); + foreach (RowCell cell in row.Cells) + { + sb.Append(cell.ColumnIndex).Append(':').Append(cell.Value.GetString()).Append('|'); + } + return sb.ToString(); + } + + private static string Rows(int count) + { + var sb = new StringBuilder(); + for (int r = 1; r <= count; r++) + { + sb.Append(CultureInfo.InvariantCulture, $"n{r}{r}"); + } + return sb.ToString(); + } + + private static byte[] Inflated(XlsxWorkbook workbook) + { + using Stream stream = workbook.OpenSheetStream(0); + using var copy = new MemoryStream(); + stream.CopyTo(copy); + return copy.ToArray(); + } + + [Fact] + public void AnXlsxChunkParsesLikeTheSameRowsReadSequentially() + { + using MemoryStream ms = WorkbookBuilder.Build(Rows(50)); + using XlsxWorkbook workbook = Excel.FromXlsx(ms); + var expected = new List(); + long dataStart; + NsTokens? ns; + using (XlsxWorkbook.Enumerator rows = workbook.OpenSheetWithoutPrefetch(0)) + { + ns = rows.Namespace; + Assert.True(rows.MoveNext()); + dataStart = rows.Offset; + while (rows.MoveNext()) + { + expected.Add(Render(rows.Current)); + } + } + byte[] inflated = Inflated(workbook); + + var actual = new List(); + using var chunk = new XlsxWorkbook.Enumerator(workbook, inflated.AsMemory((int)dataStart), ns, CancellationToken.None); + while (chunk.MoveNext()) + { + actual.Add(Render(chunk.Current)); + } + + Assert.Equal(expected, actual); + Assert.True(chunk.ReachedEnd); + Assert.False(chunk.EndedMidMarkup); + } + + [Fact] + public void AnXlsxChunkCutBetweenRowsEndsCleanly() + { + using MemoryStream ms = WorkbookBuilder.Build(Rows(10)); + using XlsxWorkbook workbook = Excel.FromXlsx(ms); + using (workbook.OpenSheetWithoutPrefetch(0)) + { + } + byte[] inflated = Inflated(workbook); + int first = inflated.AsSpan().IndexOf("1"); + using XlsxWorkbook workbook = Excel.FromXlsx(ms); + using XlsxWorkbook.Enumerator rows = workbook.OpenSheetWithoutPrefetch(0); + + Assert.NotNull(rows.Namespace); + Assert.Equal("(); + long dataStart; + bool rowOpen; + using (XlsbWorkbook.Enumerator rows = workbook.OpenSheetWithoutPrefetch(0)) + { + Assert.True(rows.MoveNext()); + dataStart = rows.Offset; + rowOpen = rows.RowOpen; + while (rows.MoveNext()) + { + expected.Add(Render(rows.Current)); + } + } + byte[] inflated; + using (Stream stream = workbook.OpenSheetStream(0)) + using (var copy = new MemoryStream()) + { + await stream.CopyToAsync(copy, ct); + inflated = copy.ToArray(); + } + + var actual = new List(); + using var chunk = new XlsbWorkbook.Enumerator(workbook, inflated.AsMemory((int)dataStart), rowOpen, CancellationToken.None); + while (chunk.MoveNext()) + { + actual.Add(Render(chunk.Current)); + } + + Assert.True(rowOpen); + Assert.Equal(29, expected.Count); + Assert.Equal(expected, actual); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Csv/CsvBlockBoundaryTests.cs b/tests/ExcelReader.Tests/Reader/Csv/CsvBlockBoundaryTests.cs index 28d22f04..f09a02ec 100644 --- a/tests/ExcelReader.Tests/Reader/Csv/CsvBlockBoundaryTests.cs +++ b/tests/ExcelReader.Tests/Reader/Csv/CsvBlockBoundaryTests.cs @@ -167,10 +167,10 @@ public void QuoteEqualToDelimiterIsRejectedByBothSourcesAlike() var options = CsvReaderOptions.Default with { Delimiter = (byte)',', Quote = (byte)',' }; byte[] bytes = Encoding.UTF8.GetBytes("a,,b,c\n,d,\ne"); - Assert.Throws(() => Excel.FromCsv(bytes, options)); + Assert.Throws(() => Excel.FromCsv(bytes, new ExcelReaderOptions { Csv = options })); using var ms = new MemoryStream(bytes); - Assert.Throws(() => Excel.FromCsv(ms, options: options)); + Assert.Throws(() => Excel.FromCsv(ms, options: new ExcelReaderOptions { Csv = options })); } } } diff --git a/tests/ExcelReader.Tests/Reader/Csv/CsvDialectAndFormatTests.cs b/tests/ExcelReader.Tests/Reader/Csv/CsvDialectAndFormatTests.cs index 8f38b216..39e20d54 100644 --- a/tests/ExcelReader.Tests/Reader/Csv/CsvDialectAndFormatTests.cs +++ b/tests/ExcelReader.Tests/Reader/Csv/CsvDialectAndFormatTests.cs @@ -135,7 +135,7 @@ public void Should_InferDelimiter_When_SniffDialectIsSet() string path = WriteTemp("Name;City;Age\nAda;London;36\n"); try { - using CsvReader reader = Excel.FromCsvFile(path, CsvReaderOptions.Default with { SniffDialect = true }); + using CsvReader reader = Excel.FromCsvFile(path, new ExcelReaderOptions { Csv = CsvReaderOptions.Default with { SniffDialect = true } }); Assert.Equal(["Ada", "London", "36"], ReadAll(reader.FirstSheet)[1]); } @@ -152,7 +152,7 @@ public void Should_KeepExplicitEncoding_When_SniffingFindsNoByteOrderMark() try { var options = CsvReaderOptions.Default with { SniffDialect = true, Encoding = Encoding.Latin1 }; - using CsvReader reader = Excel.FromCsvFile(path, options); + using CsvReader reader = Excel.FromCsvFile(path, new ExcelReaderOptions { Csv = options }); Assert.Equal(["José", "São Paulo"], ReadAll(reader.FirstSheet)[1]); } @@ -173,14 +173,11 @@ public async Task Should_InferDelimiter_When_SniffingOnTheParallelPath() string path = WriteTemp(sb.ToString()); try { - var options = new CsvParallelOptions - { - DegreeOfParallelism = 4, - Reader = CsvReaderOptions.Default with { SniffDialect = true }, - }; + var options = new SheetParallelOptions { DegreeOfParallelism = 4 }; + using CsvReader reader = Excel.FromCsvFile(path, new ExcelReaderOptions { Csv = CsvReaderOptions.Default with { SniffDialect = true } }); var rows = new List(); - await foreach (Person person in CsvParallel.ParseAsync(path, ExcelParser.FromAttributes(), options, ct: TestContext.Current.CancellationToken)) + await foreach (Person person in SheetParallel.ParseAsync(reader.FirstSheet, ExcelParser.FromAttributes(), options, ct: TestContext.Current.CancellationToken)) { rows.Add(person); } diff --git a/tests/ExcelReader.Tests/Reader/Csv/CsvLifetimeTests.cs b/tests/ExcelReader.Tests/Reader/Csv/CsvLifetimeTests.cs index 551122f0..42a986c9 100644 --- a/tests/ExcelReader.Tests/Reader/Csv/CsvLifetimeTests.cs +++ b/tests/ExcelReader.Tests/Reader/Csv/CsvLifetimeTests.cs @@ -127,7 +127,7 @@ public void A_Transcoded_Stream_Serves_Two_Enumerators() List expected = Expected(); using CsvReader reader = Excel.FromCsv( new TrickleStream(BuildCsv(Encoding.Unicode)), leaveOpen: false, - new CsvReaderOptions { Encoding = Encoding.Unicode, DetectEncodingFromByteOrderMark = false }); + new ExcelReaderOptions { Csv = new CsvReaderOptions { Encoding = Encoding.Unicode, DetectEncodingFromByteOrderMark = false } }); using CsvReader.Enumerator first = reader.FirstSheet.GetEnumerator(); using CsvReader.Enumerator second = reader.FirstSheet.GetEnumerator(); Assert.Equal(expected, Drain(first), StringComparer.Ordinal); @@ -213,7 +213,7 @@ public void A_Transcoded_File_Offers_No_Raw_Chunk_Source() File.WriteAllBytes(path, BuildCsv(Encoding.Unicode)); _paths.Add(path); - using CsvReader utf16 = Excel.FromCsvFile(path, new CsvReaderOptions { Encoding = Encoding.Unicode, DetectEncodingFromByteOrderMark = false }); + using CsvReader utf16 = Excel.FromCsvFile(path, new ExcelReaderOptions { Csv = new CsvReaderOptions { Encoding = Encoding.Unicode, DetectEncodingFromByteOrderMark = false } }); Assert.False(utf16.TryGetChunkSource(out _)); using CsvReader utf8 = Excel.FromCsvFile(path); diff --git a/tests/ExcelReader.Tests/Reader/Csv/CsvReaderTests.cs b/tests/ExcelReader.Tests/Reader/Csv/CsvReaderTests.cs index a0aac49c..771f6e33 100644 --- a/tests/ExcelReader.Tests/Reader/Csv/CsvReaderTests.cs +++ b/tests/ExcelReader.Tests/Reader/Csv/CsvReaderTests.cs @@ -292,24 +292,32 @@ public async Task QuotedFieldSplitAcrossBufferBoundaryIsHandledAsync() public void FieldExceedingMaxCellBytesThrows() { using var ms = Csv("a," + new string('x', 128 * 1024) + ",c\n"); - var options = new CsvReaderOptions { MaxCellBytes = 1024 }; - using var reader = Excel.FromCsv(ms, options: options); + using var reader = Excel.FromCsv(ms, options: new ExcelReaderOptions { MaxCellBytes = 1024 }); using CsvReader.Enumerator e = reader.FirstSheet.GetEnumerator(); ExcelLimitExceededException ex = Assert.Throws(() => e.MoveNext()); - Assert.Equal(nameof(CsvReaderOptions.MaxCellBytes), ex.LimitName); + Assert.Equal(nameof(ExcelReaderOptions.MaxCellBytes), ex.LimitName); + } + + [Fact] + public void OpenAppliesTheReaderLimitsToCsv() + { + using var ms = Csv("a," + new string('x', 128 * 1024) + ",c\n"); + using IExcelWorkbook reader = Excel.Open(ms, ExcelFileFormat.Csv, options: new ExcelReaderOptions { MaxCellBytes = 1024 }); + using IExcelRowEnumerator e = reader.FirstSheet.GetEnumerator(); + + Assert.Throws(() => e.MoveNext()); } [Fact] public void EmptyFieldsExceedingMaxCellBytesThrows() { using var ms = Csv(new string(',', 32)); - var options = new CsvReaderOptions { MaxCellBytes = 1024 }; - using var reader = Excel.FromCsv(ms, options: options); + using var reader = Excel.FromCsv(ms, options: new ExcelReaderOptions { MaxCellBytes = 1024 }); using CsvReader.Enumerator e = reader.FirstSheet.GetEnumerator(); ExcelLimitExceededException ex = Assert.Throws(() => e.MoveNext()); - Assert.Equal(nameof(CsvReaderOptions.MaxCellBytes), ex.LimitName); + Assert.Equal(nameof(ExcelReaderOptions.MaxCellBytes), ex.LimitName); } [Fact] @@ -317,7 +325,7 @@ public void SemicolonDelimiterIsRespected() { using var ms = Csv("a;b;c\n"); var options = new CsvReaderOptions { Delimiter = (byte)';' }; - using var reader = Excel.FromCsv(ms, options: options); + using var reader = Excel.FromCsv(ms, options: new ExcelReaderOptions { Csv = options }); var rows = ReadAll(reader); @@ -329,7 +337,7 @@ public void TabDelimiterIsRespected() { using var ms = Csv("a\tb\tc\n"); var options = new CsvReaderOptions { Delimiter = (byte)'\t' }; - using var reader = Excel.FromCsv(ms, options: options); + using var reader = Excel.FromCsv(ms, options: new ExcelReaderOptions { Csv = options }); var rows = ReadAll(reader); @@ -375,7 +383,7 @@ public void DelimiterEqualToQuoteThrows() using var ms = Csv("a,b\n"); var options = new CsvReaderOptions { Delimiter = (byte)'"' }; - Assert.Throws(() => Excel.FromCsv(ms, options: options)); + Assert.Throws(() => Excel.FromCsv(ms, options: new ExcelReaderOptions { Csv = options })); } [Fact] @@ -392,7 +400,7 @@ public void SeekableStreamCanBeEnumeratedMoreThanOnce() } [Fact] - public void NonSeekableStreamThrowsOnSecondEnumeration() + public void NonSeekableStreamIsReplayedOnSecondEnumeration() { byte[] bytes = Encoding.UTF8.GetBytes("a,b\n1,2\n"); using NonSeekableStream stream = new(bytes); @@ -401,7 +409,7 @@ public void NonSeekableStreamThrowsOnSecondEnumeration() var first = ReadAll(reader); Assert.Equal(2, first.Count); - Assert.Throws(reader.FirstSheet.GetEnumerator); + Assert.Equal(first, ReadAll(reader)); } @@ -487,7 +495,7 @@ public void BomDetectionCanBeDisabled() byte[] body = Encoding.UTF8.GetBytes("a,b\n"); using var ms = new MemoryStream([.. bom, .. body]); var options = new CsvReaderOptions { DetectEncodingFromByteOrderMark = false }; - using var reader = Excel.FromCsv(ms, options: options); + using var reader = Excel.FromCsv(ms, options: new ExcelReaderOptions { Csv = options }); var rows = ReadAll(reader); @@ -501,7 +509,7 @@ public async Task BomDetectionCanBeDisabledAsync() byte[] body = Encoding.UTF8.GetBytes("a,b\n"); using var ms = new MemoryStream([.. bom, .. body]); var options = new CsvReaderOptions { DetectEncodingFromByteOrderMark = false }; - using var reader = Excel.FromCsv(ms, options: options); + using var reader = Excel.FromCsv(ms, options: new ExcelReaderOptions { Csv = options }); var rows = await ReadAllAsync(reader); @@ -514,7 +522,7 @@ public void NonUtf8EncodingIsTranscoded() byte[] latin1Bytes = Encoding.Latin1.GetBytes("nome,cidade\nGabriel,São Paulo\n"); using var ms = new MemoryStream(latin1Bytes); var options = new CsvReaderOptions { Encoding = Encoding.Latin1 }; - using var reader = Excel.FromCsv(ms, options: options); + using var reader = Excel.FromCsv(ms, options: new ExcelReaderOptions { Csv = options }); var rows = ReadAll(reader); @@ -529,7 +537,7 @@ public void DelimiterAsLineTerminatorThrows(byte delimiter) using var ms = Csv("a,b\n"); var options = new CsvReaderOptions { Delimiter = delimiter }; - Assert.Throws(() => Excel.FromCsv(ms, options: options)); + Assert.Throws(() => Excel.FromCsv(ms, options: new ExcelReaderOptions { Csv = options })); } [Theory] @@ -540,7 +548,7 @@ public void QuoteAsLineTerminatorThrows(byte quote) using var ms = Csv("a,b\n"); var options = new CsvReaderOptions { Quote = quote }; - Assert.Throws(() => Excel.FromCsv(ms, options: options)); + Assert.Throws(() => Excel.FromCsv(ms, options: new ExcelReaderOptions { Csv = options })); } diff --git a/tests/ExcelReader.Tests/Reader/Csv/CsvReplayTests.cs b/tests/ExcelReader.Tests/Reader/Csv/CsvReplayTests.cs new file mode 100644 index 00000000..e68f5592 --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Csv/CsvReplayTests.cs @@ -0,0 +1,159 @@ +using System.Globalization; +using System.Text; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Csv; +using ExcelReader.Core.Reader.Schema; + +namespace ExcelReader.Tests.Reader.Csv +{ + public sealed class CsvReplayTests + { + private static byte[] BuildCsv(int rows, Encoding? encoding = null) + { + StringBuilder text = new("name,value\n"); + for (int r = 0; r < rows; r++) + { + text.Append("name-").Append(r.ToString(CultureInfo.InvariantCulture)) + .Append(',').Append((r * 7).ToString(CultureInfo.InvariantCulture)).Append('\n'); + } + return (encoding ?? Encoding.UTF8).GetBytes(text.ToString()); + } + + private static List Drain(CsvReader reader) + { + List lines = []; + using CsvReader.Enumerator e = reader.FirstSheet.GetEnumerator(); + while (e.MoveNext()) + { + Row row = e.Current; + lines.Add(row[0].GetString() + "|" + row[1].GetString()); + } + return lines; + } + + private static List Expected(int rows) + { + using CsvReader reference = Excel.FromCsv(BuildCsv(rows)); + return Drain(reference); + } + + [Fact] + public void A_Schema_Inferred_From_A_NonSeekable_Stream_Leaves_Every_Row_To_Read() + { + const int rows = 20_000; + using NonSeekableStream stream = new(BuildCsv(rows)); + using CsvReader reader = Excel.FromCsv(stream); + + IReadOnlyList schema = Excel.InferSchema(reader.FirstSheet, new SchemaInferenceOptions { ParseText = true }); + + Assert.Equal(["name", "value"], schema.Select(c => c.Name), StringComparer.Ordinal); + Assert.Equal(ExcelColumnType.Int64Column, schema[1].Type); + Assert.Equal(Expected(rows), Drain(reader), StringComparer.Ordinal); + } + + [Fact] + public void A_Small_NonSeekable_Stream_Can_Be_Read_Any_Number_Of_Times() + { + const int rows = 500; + List expected = Expected(rows); + using NonSeekableStream stream = new(BuildCsv(rows)); + using CsvReader reader = Excel.FromCsv(stream); + + Assert.Equal(expected, Drain(reader), StringComparer.Ordinal); + Assert.Equal(expected, Drain(reader), StringComparer.Ordinal); + Assert.Equal(expected, Drain(reader), StringComparer.Ordinal); + } + + [Fact] + public void A_Transcoded_NonSeekable_Stream_Is_Replayed_Too() + { + const int rows = 20_000; + using NonSeekableStream stream = new(BuildCsv(rows, Encoding.Unicode)); + using CsvReader reader = Excel.FromCsv( + stream, leaveOpen: false, + new ExcelReaderOptions { Csv = new CsvReaderOptions { Encoding = Encoding.Unicode, DetectEncodingFromByteOrderMark = false } }); + + Excel.InferSchema(reader.FirstSheet); + + Assert.Equal(Expected(rows), Drain(reader), StringComparer.Ordinal); + } + + [Fact] + public void A_Second_Enumeration_Is_Refused_While_The_First_Is_Open() + { + using NonSeekableStream stream = new(BuildCsv(10)); + using CsvReader reader = Excel.FromCsv(stream); + using CsvReader.Enumerator first = reader.FirstSheet.GetEnumerator(); + + Assert.Throws(reader.FirstSheet.GetEnumerator); + } + + [Fact] + public void A_Stream_Read_Past_The_Replay_Limit_Cannot_Be_Read_Again() + { + byte[] line = Encoding.UTF8.GetBytes(new string('x', 1021) + ",1\n"); + byte[] bytes = new byte[CsvReader.ReplayLimit + (line.Length * 4)]; + for (int offset = 0; offset < bytes.Length; offset += line.Length) + { + line.CopyTo(bytes, offset); + } + using NonSeekableStream stream = new(bytes); + using CsvReader reader = Excel.FromCsv(stream); + + Assert.Equal(bytes.Length / line.Length, Drain(reader).Count); + + InvalidOperationException error = Assert.Throws(reader.FirstSheet.GetEnumerator); + Assert.Contains("cannot be enumerated again", error.Message, StringComparison.Ordinal); + } + + [Fact] + public void A_Pass_Replays_What_Was_Read_And_Then_Continues() + { + byte[] bytes = Enumerable.Range(0, 100).Select(i => (byte)i).ToArray(); + using NonSeekableStream inner = new(bytes); + using ReplayStream replay = new(inner, leaveOpen: true, limit: 64); + + Assert.True(replay.TryBeginPass()); + byte[] head = new byte[40]; + replay.ReadExactly(head); + replay.Dispose(); + + Assert.True(replay.TryBeginPass()); + byte[] all = new byte[100]; + replay.ReadExactly(all); + Assert.Equal(bytes, all); + Assert.Equal(0, replay.Read(new byte[1])); + replay.Dispose(); + + Assert.False(replay.TryBeginPass()); + } + + [Fact] + public void Releasing_Closes_The_Inner_Stream_Only_When_It_Is_Owned() + { + using TrackingStream owned = new(); + using TrackingStream borrowed = new(); + ReplayStream a = new(owned, leaveOpen: false, limit: 8); + ReplayStream b = new(borrowed, leaveOpen: true, limit: 8); + + a.Dispose(); + Assert.False(owned.Disposed); + a.Release(); + b.Release(); + + Assert.True(owned.Disposed); + Assert.False(borrowed.Disposed); + } + + private sealed class TrackingStream : MemoryStream + { + internal bool Disposed { get; private set; } + + protected override void Dispose(bool disposing) + { + Disposed = true; + base.Dispose(disposing); + } + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Csv/CsvSnifferTests.cs b/tests/ExcelReader.Tests/Reader/Csv/CsvSnifferTests.cs index b4620e2a..6820038a 100644 --- a/tests/ExcelReader.Tests/Reader/Csv/CsvSnifferTests.cs +++ b/tests/ExcelReader.Tests/Reader/Csv/CsvSnifferTests.cs @@ -177,7 +177,7 @@ public void SniffedDialectRoundTripsThroughReader() byte[] data = Bytes("nome;idade\nAna;30\nBia;25\n"); CsvDialect dialect = CsvSniffer.Detect(data); CsvReaderOptions options = CsvReaderOptions.Default.WithDialect(dialect); - using var reader = Excel.FromCsv(data, options); + using var reader = Excel.FromCsv(data, new ExcelReaderOptions { Csv = options }); using CsvReader.Enumerator e = reader.FirstSheet.GetEnumerator(); Assert.True(e.MoveNext()); diff --git a/tests/ExcelReader.Tests/Reader/FastDateTests.cs b/tests/ExcelReader.Tests/Reader/FastDateTests.cs index b4833bd3..b8edd98e 100644 --- a/tests/ExcelReader.Tests/Reader/FastDateTests.cs +++ b/tests/ExcelReader.Tests/Reader/FastDateTests.cs @@ -146,7 +146,7 @@ public void CsvTextDatesAreReadableThroughTryGetDateTime() { if (col == 1) { - Assert.True(rowCell.Value.TryGetDateTime(out DateTime d)); + Assert.True(rowCell.Value.TryGetDateTime(isDate1904: false, out DateTime d)); dates.Add(d); } col++; @@ -165,7 +165,7 @@ public void NumericCellsStillReadAsExcelSerials() foreach (Row row in reader.FirstSheet) { - Assert.True(row[1].TryGetDateTime(out DateTime d)); + Assert.True(row[1].TryGetDateTime(isDate1904: false, out DateTime d)); Assert.Equal(new DateTime(2024, 3, 15), d); } } diff --git a/tests/ExcelReader.Tests/Reader/FileOwnershipTests.cs b/tests/ExcelReader.Tests/Reader/FileOwnershipTests.cs index e0db4145..f680d5d7 100644 --- a/tests/ExcelReader.Tests/Reader/FileOwnershipTests.cs +++ b/tests/ExcelReader.Tests/Reader/FileOwnershipTests.cs @@ -79,10 +79,10 @@ private async Task Open(string entryPoint) Excel.FromXlsbFile(_path).Dispose(); break; case "FromCsvFile": - Excel.FromCsvFile(_path, DelimiterEqualsQuote).Dispose(); + Excel.FromCsvFile(_path, new ExcelReaderOptions { Csv = DelimiterEqualsQuote }).Dispose(); break; case "FromCsvFileAsync": - (await Excel.FromCsvFileAsync(_path, DelimiterEqualsQuote)).Dispose(); + (await Excel.FromCsvFileAsync(_path, new ExcelReaderOptions { Csv = DelimiterEqualsQuote })).Dispose(); break; default: (await Excel.FromCsvFileAsync(_path, ct: new CancellationToken(canceled: true))).Dispose(); diff --git a/tests/ExcelReader.Tests/Reader/ReaderLimitTests.cs b/tests/ExcelReader.Tests/Reader/ReaderLimitTests.cs index 459e19eb..babf22dd 100644 --- a/tests/ExcelReader.Tests/Reader/ReaderLimitTests.cs +++ b/tests/ExcelReader.Tests/Reader/ReaderLimitTests.cs @@ -6,15 +6,56 @@ using ExcelReader.Core.Crypto; using ExcelReader.Core.Reader; using ExcelReader.Core.Reader.Internal; +using ExcelReader.Core.Reader.Xlsb; using ExcelReader.Core.Reader.Xlsx; using ExcelReader.Core.Reader.Zip; +using ExcelReader.Core.Writer.Xlsb; +using ExcelReader.Core.Writer.Xlsx; using ExcelReader.Tests.Crypto; +using ExcelReader.Tests.Reader.Tables; using ExcelReader.Tests.Reader.Xls; +using B = ExcelReader.Tests.Reader.Xlsb.Biff12Build; namespace ExcelReader.Tests.Reader { public class ReaderLimitTests { + [Theory] + [InlineData("A1:B2", "")] + [InlineData("nonsense", "")] + [InlineData("A0:A2", "")] + [InlineData("A1:A2", " headerRowCount=\"2\"")] + [InlineData("A1:A2", " totalsRowCount=\"x\"")] + [InlineData("A1:A1", " totalsRowCount=\"1\"")] + public void MalformedXlsxTablePartThrowsInvalidData(string reference, string attributes) + { + using MemoryStream ms = TableWorkbooks.Xlsx( + """1""", + TableWorkbooks.Table(reference, """""", attributes)); + + Assert.Throws(() => Excel.FromXlsx(ms)); + } + + [Fact] + public void XlsxTablePartWithoutATableElementThrowsInvalidData() + { + using MemoryStream ms = TableWorkbooks.Xlsx("""1""", ""); + + Assert.Throws(() => Excel.FromXlsx(ms)); + } + + [Fact] + public void XlsxTableRelationshipToAMissingPartThrowsInvalidData() + { + using MemoryStream ms = TableWorkbooks.Xlsx( + """1""", + TableWorkbooks.Table("A1:A1", """"""), + target: "../tables/table9.xml"); + + InvalidDataException ex = Assert.Throws(() => Excel.FromXlsx(ms)); + Assert.Contains("xl/tables/table9.xml", ex.Message, StringComparison.Ordinal); + } + [Fact] public void CellAccumulatorRejectsColumnIndexAtOrAboveExcelLimit() { @@ -352,11 +393,99 @@ public void ZeroTotalDecompressedLimitRestoresUnlimitedTotalRead() Assert.Equal(value.Length, e.Current[0].GetString().Length); } + private const int BudgetRows = 5_000; + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, false)] + [InlineData(true, true)] + public void RereadingASheetStartsAFreshTotalDecompressedBudget(bool xlsb, bool fromStream) + { + byte[] package = WriteTextRows(xlsb, sharedStrings: false); + (long total, long sheet) = EntrySizes(package, xlsb); + var options = new ExcelReaderOptions { MaxTotalDecompressedBytes = total + (sheet / 2) }; + + using MemoryStream stream = new(package); + using IExcelWorkbook workbook = fromStream + ? Excel.Open(stream, options: options) + : Excel.Open(package, options); + for (int pass = 0; pass < 3; pass++) + { + Assert.Equal(BudgetRows, CountRows(workbook.FirstSheet)); + } + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void OneSheetReadStillCountsTheSharedStringsTowardTheTotalBudget(bool xlsb) + { + byte[] package = WriteTextRows(xlsb, sharedStrings: true); + (long total, long sheet) = EntrySizes(package, xlsb); + var options = new ExcelReaderOptions { MaxTotalDecompressedBytes = total - (sheet / 2) }; + + using IExcelWorkbook workbook = Excel.Open(package, options); + ExcelLimitExceededException ex = Assert.Throws(() => CountRows(workbook.FirstSheet)); + Assert.Equal(nameof(ExcelReaderOptions.MaxTotalDecompressedBytes), ex.LimitName); + } + + private static byte[] WriteTextRows(bool xlsb, bool sharedStrings) + { + using MemoryStream ms = new(); + if (xlsb) + { + using XlsbWorkbookWriter workbook = XlsbWorkbookWriter.Create(ms, leaveOpen: true, new XlsbWriterOptions { UseSharedStrings = sharedStrings }); + using (XlsbSheetWriter sheet = workbook.AddSheet("S")) + { + for (int i = 0; i < BudgetRows; i++) + { + using XlsbRowWriter row = sheet.StartRow(); + row.Write(string.Create(CultureInfo.InvariantCulture, $"value {i:D6} padding padding padding")); + } + } + workbook.End(); + } + else + { + using XlsxWorkbookWriter workbook = XlsxWorkbookWriter.Create(ms, leaveOpen: true, new XlsxWriterOptions { UseSharedStrings = sharedStrings }); + using (XlsxSheetWriter sheet = workbook.AddSheet("S")) + { + for (int i = 0; i < BudgetRows; i++) + { + using XlsxRowWriter row = sheet.StartRow(); + row.Write(string.Create(CultureInfo.InvariantCulture, $"value {i:D6} padding padding padding")); + } + } + workbook.End(); + } + return ms.ToArray(); + } + + private static (long Total, long Sheet) EntrySizes(byte[] package, bool xlsb) + { + using MemoryStream ms = new(package); + using ZipArchive zip = new(ms, ZipArchiveMode.Read); + long total = zip.Entries.Sum(entry => entry.Length); + long sheet = zip.GetEntry(xlsb ? "xl/worksheets/sheet1.bin" : "xl/worksheets/sheet1.xml")!.Length; + return (total, sheet); + } + + private static int CountRows(IExcelSheet sheet) + { + int rows = 0; + foreach (Row row in sheet) + { + rows++; + } + return rows; + } + [Fact] public async Task LimitedReadStreamCoversUnsupportedOperationsAndByteArrayAsyncRead() { await using MemoryStream inner = new([1, 2, 3, 4, 5]); - await using LimitedReadStream limited = new(inner, new DecompressedByteCounter(limit: 10)); + await using LimitedReadStream limited = new(inner, declaredLength: 5, new DecompressedByteCounter(limit: 10)); Assert.True(limited.CanRead); Assert.False(limited.CanSeek); @@ -382,6 +511,7 @@ public void LimitedReadStreamCountsSpanReadsAndEntryLimit() using MemoryStream inner = new([1, 2, 3, 4]); using LimitedReadStream limited = new( inner, + declaredLength: 4, totalCounter: null, entryLimitName: nameof(ExcelReaderOptions.MaxCellBytes), entryLimit: 2); @@ -608,5 +738,89 @@ public void MiniStreamChainWithACycleThrowsInvalidDataInsteadOfRepeatingSectors( Assert.Throws(() => CfbContainer.ReadMiniStream(miniStream, miniFat, miniSectorSize: 64, startSector: 0, size: 256)); } + + [Fact] + public void SyntheticXlsbTableParses() + { + using MemoryStream ms = TableWorkbooks.Xlsb( + [ + .. TableWorkbooks.BeginList(0, 1, 0, 0, 1, 0, "T"), + .. TableWorkbooks.ListColumn(1, "H"), + .. TableWorkbooks.StyleClient("TableStyleLight1"), + ]); + using XlsbWorkbook workbook = Excel.FromXlsb(ms); + + ExcelTable table = Assert.Single(workbook.Tables); + Assert.Equal(("T", "A1:A2", "TableStyleLight1"), (table.Name, table.Ref, table.StyleName)); + Assert.Equal(["H"], table.ColumnNames); + } + + [Fact] + public void TruncatedXlsbBeginListThrowsInvalidData() + { + using MemoryStream ms = TableWorkbooks.Xlsb(B.Record(Brt.BeginList, new byte[10])); + + Assert.Throws(() => Excel.FromXlsb(ms)); + } + + [Fact] + public void XlsbTablePartWithoutBeginListThrowsInvalidData() + { + using MemoryStream ms = TableWorkbooks.Xlsb(TableWorkbooks.ListColumn(1, "H")); + + Assert.Throws(() => Excel.FromXlsb(ms)); + } + + [Fact] + public void XlsbTableWithNegativeRowsThrowsInvalidData() + { + using MemoryStream ms = TableWorkbooks.Xlsb( + [ + .. TableWorkbooks.BeginList(0xFFFFFFFF, 1, 0, 0, 1, 0, "T"), + .. TableWorkbooks.ListColumn(1, "H"), + ]); + + Assert.Throws(() => Excel.FromXlsb(ms)); + } + + [Theory] + [InlineData(0xFFFFFFFFu)] + [InlineData(0u)] + public void XlsbTableOverMalformedRowNumbersThrowsInvalidData(uint badRow) + { + using MemoryStream ms = TableWorkbooks.Xlsb( + [.. TableWorkbooks.BeginList(0, 3, 0, 0, 1, 0, "T"), .. TableWorkbooks.ListColumn(1, "H")], + [ + .. B.Record(Brt.RowHdr, B.U32(0)), + .. B.Record(Brt.CellSt, B.CellSt(0, 0, "H")), + .. B.Record(Brt.RowHdr, B.U32(1)), + .. B.Record(Brt.CellRk, B.CellRk(0, 0, (1u << 2) | 0x02)), + .. B.Record(Brt.RowHdr, B.U32(badRow)), + .. B.Record(Brt.CellRk, B.CellRk(0, 0, (2u << 2) | 0x02)), + .. B.Record(Brt.RowHdr, B.U32(2)), + .. B.Record(Brt.CellRk, B.CellRk(0, 0, (3u << 2) | 0x02)), + .. B.Record(Brt.EndSheetData), + ]); + using XlsbWorkbook workbook = Excel.FromXlsb(ms); + using IExcelRowEnumerator rows = Assert.Single(workbook.Tables).AsSheet().GetEnumerator(); + + Assert.Throws(() => + { + while (rows.MoveNext()) + { + } + }); + } + + [Fact] + public async Task MalformedXlsbTableFromStreamingSourceThrowsInvalidData() + { + byte[] bytes = TableWorkbooks.Xlsb(TableWorkbooks.ListColumn(1, "H")).ToArray(); + + Assert.Throws(() => Excel.FromXlsb(new StreamBackedMemory(bytes))); + await Assert.ThrowsAsync(async () => await Excel.FromXlsbAsync(new StreamBackedMemory(bytes), leaveOpen: false, ct: TestContext.Current.CancellationToken)); + } + + private sealed class StreamBackedMemory(byte[] bytes) : MemoryStream(bytes); } } diff --git a/tests/ExcelReader.Tests/Reader/SampleTest.cs b/tests/ExcelReader.Tests/Reader/SampleTest.cs index 86e0bddc..590755d9 100644 --- a/tests/ExcelReader.Tests/Reader/SampleTest.cs +++ b/tests/ExcelReader.Tests/Reader/SampleTest.cs @@ -135,11 +135,11 @@ public void DetectsDateStylesAndConvertsSerial() var row = e.Current; Assert.Equal(CellType.Date, row[0].Type); - Assert.True(row[0].TryGetDateTime(out var d0)); + Assert.True(row[0].TryGetDateTime(isDate1904: false, out var d0)); Assert.Equal(new DateTime(2024, 1, 1, 0, 0, 0, DateTimeKind.Unspecified), d0); Assert.Equal(CellType.Date, row[1].Type); - Assert.True(row[1].TryGetDateTime(out var d1)); + Assert.True(row[1].TryGetDateTime(isDate1904: false, out var d1)); Assert.Equal(new DateTime(2024, 1, 1, 12, 0, 0, DateTimeKind.Unspecified), d1); Assert.Equal(CellType.Number, row[2].Type); @@ -188,7 +188,7 @@ public async Task AsyncHandlesBufferGrowthAndDates() Assert.True(await e.MoveNextAsync()); Assert.Equal(CellType.Date, e.Current[0].Type); - Assert.True(e.Current[0].TryGetDateTime(out var d)); + Assert.True(e.Current[0].TryGetDateTime(isDate1904: false, out var d)); Assert.Equal(new DateTime(2024, 1, 1, 0, 0, 0, DateTimeKind.Unspecified), d); Assert.True(await e.MoveNextAsync()); diff --git a/tests/ExcelReader.Tests/Reader/SchemaInferenceTests.cs b/tests/ExcelReader.Tests/Reader/SchemaInferenceTests.cs index 984a013c..d9ca2f9e 100644 --- a/tests/ExcelReader.Tests/Reader/SchemaInferenceTests.cs +++ b/tests/ExcelReader.Tests/Reader/SchemaInferenceTests.cs @@ -45,9 +45,9 @@ private static ExcelColumnSchema[] InferFromXlsx(string sheetRows, int headerRow return SchemaInference.Infer(rows, isDate1904: false, headerRow, sampleSize, parseText); } - private static ExcelColumnType[] TypesOf(ExcelColumnSchema[] schema) + private static ExcelColumnType[] TypesOf(IReadOnlyList schema) { - return Array.ConvertAll(schema, column => column.Type); + return [.. schema.Select(column => column.Type)]; } private static string FixtureCsv() @@ -67,7 +67,7 @@ public void InferSchema_Should_Type_The_Fixture_Csv_When_Parsing_Text() { using CsvReader reader = Excel.FromCsvFile(FixtureCsv()); - ExcelColumnSchema[] schema = Excel.InferSchema(reader.FirstSheet, parseText: true); + IReadOnlyList schema = Excel.InferSchema(reader.FirstSheet, new SchemaInferenceOptions { ParseText = true }); ExcelColumnType s = ExcelColumnType.StringColumn, d = ExcelColumnType.DateColumn, i = ExcelColumnType.Int64Column, f = ExcelColumnType.Float64Column; @@ -292,11 +292,11 @@ public void Should_SampleAtMostSampleSizeRows_When_TheSheetIsLonger() public void Should_InferAcrossEveryFormat_When_CalledThroughTheExcelFacade() { using IExcelWorkbook reader = Excel.Open(Path.Combine("data", "RealExcel.xlsb")); - ExcelColumnSchema[] schema = Excel.InferSchema(reader.FirstSheet); + IReadOnlyList schema = Excel.InferSchema(reader.FirstSheet); Assert.NotEmpty(schema); Assert.All(schema, column => Assert.True(column.Index >= 0)); - for (int i = 0; i < schema.Length; i++) + for (int i = 0; i < schema.Count; i++) { Assert.Equal(i, schema[i].Index); } diff --git a/tests/ExcelReader.Tests/Reader/SheetConsumerTests.cs b/tests/ExcelReader.Tests/Reader/SheetConsumerTests.cs index 88b396ec..3c100dff 100644 --- a/tests/ExcelReader.Tests/Reader/SheetConsumerTests.cs +++ b/tests/ExcelReader.Tests/Reader/SheetConsumerTests.cs @@ -158,25 +158,25 @@ public void One_Parse_Result_Can_Be_Enumerated_Twice_And_Sheets_Stay_Independent public void InferSchema_Samples_The_Sheet_It_Is_Given() { using XlsxWorkbook workbook = Excel.FromXlsx(BuildXlsx()); - ExcelColumnSchema[] schema = Excel.InferSchema(workbook.Sheets[1]); - Assert.Equal(2, schema.Length); + IReadOnlyList schema = Excel.InferSchema(workbook.Sheets[1]); + Assert.Equal(2, schema.Count); Assert.Equal("Name", schema[0].Name); Assert.Equal(ExcelColumnType.Int64Column, schema[1].Type); - Assert.Equal(schema.Length, Excel.InferSchema(workbook.Sheets[1], 1, 5).Length); + Assert.Equal(schema.Count, Excel.InferSchema(workbook.Sheets[1], new SchemaInferenceOptions { SampleSize = 5 }).Count); Assert.Throws(() => Excel.InferSchema((IExcelSheet)null!)); } [Fact] - public void ExcelDataReader_Reads_The_Sheet_It_Is_Given_And_Names_Its_Table_After_It() + public void SheetDataReader_Reads_The_Sheet_It_Is_Given_And_Names_Its_Table_After_It() { using XlsxWorkbook workbook = Excel.FromXlsx(BuildXlsx()); - using ExcelDataReader data = new(workbook.Sheets[1]); + using SheetDataReader data = new(workbook.Sheets[1]); Assert.Equal(2, data.FieldCount); Assert.True(data.Read()); Assert.Equal("s1-0", data.GetString(0)); using DataTable schema = data.GetSchemaTable()!; Assert.Equal("S1", schema.Rows[0][SchemaTableColumn.BaseTableName]); - Assert.Throws(() => new ExcelDataReader((IExcelSheet)null!)); + Assert.Throws(() => new SheetDataReader((IExcelSheet)null!)); } [Fact] diff --git a/tests/ExcelReader.Tests/Reader/ExcelDataReaderTests.cs b/tests/ExcelReader.Tests/Reader/SheetDataReaderTests.cs similarity index 90% rename from tests/ExcelReader.Tests/Reader/ExcelDataReaderTests.cs rename to tests/ExcelReader.Tests/Reader/SheetDataReaderTests.cs index bc4c7f62..e725b8f9 100644 --- a/tests/ExcelReader.Tests/Reader/ExcelDataReaderTests.cs +++ b/tests/ExcelReader.Tests/Reader/SheetDataReaderTests.cs @@ -9,7 +9,7 @@ namespace ExcelReader.Tests.Reader { - public class ExcelDataReaderTests + public class SheetDataReaderTests { [Fact] public async Task HeaderRowFixesNamesAndFieldCount() @@ -18,7 +18,7 @@ public async Task HeaderRowFixesNamesAndFieldCount() ["Id", "Name"], [1, "alice"]); await using IExcelWorkbook reader = await Excel.OpenAsync(ms, ct: TestContext.Current.CancellationToken); - using var data = new ExcelDataReader(reader.FirstSheet); + using var data = new SheetDataReader(reader.FirstSheet); Assert.Equal(2, data.FieldCount); Assert.Equal("Id", data.GetName(0)); @@ -36,7 +36,7 @@ public async Task ReadWalksEveryDataRowAfterTheHeader() [2], [3]); await using IExcelWorkbook reader = await Excel.OpenAsync(ms, ct: TestContext.Current.CancellationToken); - using var data = new ExcelDataReader(reader.FirstSheet); + using var data = new SheetDataReader(reader.FirstSheet); var seen = new List(); while (data.Read()) @@ -53,7 +53,7 @@ public async Task GetValueMapsEachCellTypeToItsClrType() ["Text", "Number", "Flag", "When", "Blank"], ["hi", 42, true, new DateTime(2024, 1, 1), null]); await using IExcelWorkbook reader = await Excel.OpenAsync(ms, ct: TestContext.Current.CancellationToken); - using var data = new ExcelDataReader(reader.FirstSheet); + using var data = new SheetDataReader(reader.FirstSheet); Assert.True(data.Read()); Assert.Equal("hi", data.GetValue(0)); @@ -69,7 +69,7 @@ public async Task NoHeaderSynthesizesColumnNamesFromFirstRowWidth() { await using var ms = await TypedWorkbook.BuildAsync([1, 2, 3]); await using IExcelWorkbook reader = await Excel.OpenAsync(ms, ct: TestContext.Current.CancellationToken); - using var data = new ExcelDataReader(reader.FirstSheet, headerRow: 0); + using var data = new SheetDataReader(reader.FirstSheet, new SheetDataReaderOptions { HeaderRow = 0 }); Assert.Equal(3, data.FieldCount); Assert.Equal("Column0", data.GetName(0)); @@ -83,7 +83,7 @@ public async Task HeaderRowBeyondSheetSizeIsAnEmptyResultSet() { await using var ms = await TypedWorkbook.BuildAsync(["Id"]); await using IExcelWorkbook reader = await Excel.OpenAsync(ms, ct: TestContext.Current.CancellationToken); - using var data = new ExcelDataReader(reader.FirstSheet, headerRow: 5); + using var data = new SheetDataReader(reader.FirstSheet, new SheetDataReaderOptions { HeaderRow = 5 }); Assert.Equal(0, data.FieldCount); Assert.False(data.Read()); @@ -94,7 +94,7 @@ public async Task GetOrdinalForUnknownNameThrows() { await using var ms = await TypedWorkbook.BuildAsync(["Id"], [1]); await using IExcelWorkbook reader = await Excel.OpenAsync(ms, ct: TestContext.Current.CancellationToken); - using var data = new ExcelDataReader(reader.FirstSheet); + using var data = new SheetDataReader(reader.FirstSheet); Assert.Throws(() => data.GetOrdinal("DoesNotExist")); } @@ -104,7 +104,7 @@ public async Task GetValueBeforeReadThrows() { await using var ms = await TypedWorkbook.BuildAsync(["Id"], [1]); await using IExcelWorkbook reader = await Excel.OpenAsync(ms, ct: TestContext.Current.CancellationToken); - using var data = new ExcelDataReader(reader.FirstSheet); + using var data = new SheetDataReader(reader.FirstSheet); Assert.Throws(() => data.GetValue(0)); } @@ -116,7 +116,7 @@ public async Task NextResultAlwaysReturnsFalse() ("A", [[1]]), ("B", [[2]])); await using IExcelWorkbook reader = await Excel.OpenAsync(ms, ct: TestContext.Current.CancellationToken); - using var data = new ExcelDataReader(reader.FirstSheet); + using var data = new SheetDataReader(reader.FirstSheet); Assert.False(data.NextResult()); } @@ -126,7 +126,7 @@ public async Task GetBytesReturnsTheCellsUtf8BytesWindowedByOffsetAndLength() { await using var ms = await TypedWorkbook.BuildAsync(["Text"], ["héllo"]); await using IExcelWorkbook reader = await Excel.OpenAsync(ms, ct: TestContext.Current.CancellationToken); - using var data = new ExcelDataReader(reader.FirstSheet); + using var data = new SheetDataReader(reader.FirstSheet); Assert.True(data.Read()); byte[] expected = Encoding.UTF8.GetBytes("héllo"); @@ -148,7 +148,7 @@ public async Task GetBytesFormatsANumericCellInsteadOfReturningNothing() { await using var ms = await TypedWorkbook.BuildAsync(["Number"], [42]); await using IExcelWorkbook reader = await Excel.OpenAsync(ms, ct: TestContext.Current.CancellationToken); - using var data = new ExcelDataReader(reader.FirstSheet); + using var data = new SheetDataReader(reader.FirstSheet); Assert.True(data.Read()); byte[] buffer = new byte[8]; @@ -161,7 +161,7 @@ public async Task GetBytesOnADbNullCellReturnsZero() { await using var ms = await TypedWorkbook.BuildAsync(["Blank"], [null]); await using IExcelWorkbook reader = await Excel.OpenAsync(ms, ct: TestContext.Current.CancellationToken); - using var data = new ExcelDataReader(reader.FirstSheet); + using var data = new SheetDataReader(reader.FirstSheet); Assert.True(data.Read()); Assert.Equal(0, data.GetBytes(0, 0, null, 0, 0)); @@ -174,7 +174,7 @@ public async Task GetCharsReturnsTheCellsTextWindowedByOffsetAndLength() { await using var ms = await TypedWorkbook.BuildAsync(["Text"], ["hello world"]); await using IExcelWorkbook reader = await Excel.OpenAsync(ms, ct: TestContext.Current.CancellationToken); - using var data = new ExcelDataReader(reader.FirstSheet); + using var data = new SheetDataReader(reader.FirstSheet); Assert.True(data.Read()); Assert.Equal(11, data.GetChars(0, 0, null, 0, 0)); @@ -206,7 +206,7 @@ public async Task GetBytesFormatsAnXlsbNumericCellFromItsBinaryValue() ms.Position = 0; using XlsbWorkbook reader = Excel.FromXlsb(ms); - using var data = new ExcelDataReader(reader.FirstSheet); + using var data = new SheetDataReader(reader.FirstSheet); Assert.True(data.Read()); byte[] buffer = new byte[8]; @@ -219,7 +219,7 @@ public async Task TypedGettersParseANumericCellIntoEveryIntegralAndFloatingType( { await using var ms = await TypedWorkbook.BuildAsync(["Number"], [42]); await using IExcelWorkbook reader = await Excel.OpenAsync(ms, ct: TestContext.Current.CancellationToken); - using var data = new ExcelDataReader(reader.FirstSheet); + using var data = new SheetDataReader(reader.FirstSheet); Assert.True(data.Read()); Assert.Equal((byte)42, data.GetByte(0)); @@ -238,7 +238,7 @@ public async Task GetBooleanAndGetDateTimeReadTheirOwnCellTypesAndRejectText() ["Flag", "When", "Text"], [true, new DateTime(2024, 3, 7), "nope"]); await using IExcelWorkbook reader = await Excel.OpenAsync(ms, ct: TestContext.Current.CancellationToken); - using var data = new ExcelDataReader(reader.FirstSheet); + using var data = new SheetDataReader(reader.FirstSheet); Assert.True(data.Read()); Assert.True(data.GetBoolean(0)); @@ -255,7 +255,7 @@ public async Task GetGuidParsesATextCellAndRejectsOneThatIsNotAGuid() ["Id", "Text"], [guid.ToString(), "nope"]); await using IExcelWorkbook reader = await Excel.OpenAsync(ms, ct: TestContext.Current.CancellationToken); - using var data = new ExcelDataReader(reader.FirstSheet); + using var data = new SheetDataReader(reader.FirstSheet); Assert.True(data.Read()); Assert.Equal(guid, data.GetGuid(0)); @@ -269,7 +269,7 @@ public async Task GetCharTakesTheFirstCharacterAndRejectsAnEmptyCell() ["Text", "Blank"], ["hello", null]); await using IExcelWorkbook reader = await Excel.OpenAsync(ms, ct: TestContext.Current.CancellationToken); - using var data = new ExcelDataReader(reader.FirstSheet); + using var data = new SheetDataReader(reader.FirstSheet); Assert.True(data.Read()); Assert.Equal('h', data.GetChar(0)); @@ -283,7 +283,7 @@ public async Task GetFieldTypeFollowsTheCurrentRowsOwnCellTypes() ["Text", "Number", "Flag", "When", "Blank"], ["hi", 42, true, new DateTime(2024, 1, 1), null]); await using IExcelWorkbook reader = await Excel.OpenAsync(ms, ct: TestContext.Current.CancellationToken); - using var data = new ExcelDataReader(reader.FirstSheet); + using var data = new SheetDataReader(reader.FirstSheet); Assert.True(data.Read()); Assert.Equal(typeof(string), data.GetFieldType(0)); @@ -299,7 +299,7 @@ public async Task IndexersAndTheFixedAdoMembersReportTheDocumentedValues() { await using var ms = await TypedWorkbook.BuildAsync(["Id", "Name"], [1, "alice"]); await using IExcelWorkbook reader = await Excel.OpenAsync(ms, ct: TestContext.Current.CancellationToken); - using var data = new ExcelDataReader(reader.FirstSheet); + using var data = new SheetDataReader(reader.FirstSheet); Assert.True(data.Read()); Assert.Equal(1.0, data[0]); @@ -317,7 +317,7 @@ public async Task DataTableLoadPopulatesFromTheSheet() [1, "alice"], [2, "bob"]); await using IExcelWorkbook reader = await Excel.OpenAsync(ms, ct: TestContext.Current.CancellationToken); - using var data = new ExcelDataReader(reader.FirstSheet); + using var data = new SheetDataReader(reader.FirstSheet); var table = new DataTable(); table.Load(data); @@ -353,7 +353,7 @@ public void AFailedConstructorDoesNotKeepTheReaderAlive() XlsxWorkbook reader = Excel.FromXlsx(stream, leaveOpen: false, new ExcelReaderOptions { MaxCellBytes = 64 }); Assert.Throws(() => { - using ExcelDataReader data = new(reader.FirstSheet); + using SheetDataReader data = new(reader.FirstSheet); }); reader.Dispose(); Assert.False(stream.CanRead); diff --git a/tests/ExcelReader.Tests/Reader/Sources/CachedByteSourceTests.cs b/tests/ExcelReader.Tests/Reader/Sources/CachedByteSourceTests.cs new file mode 100644 index 00000000..e8d81494 --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Sources/CachedByteSourceTests.cs @@ -0,0 +1,89 @@ +using ExcelReader.Core.Reader.Sources; + +namespace ExcelReader.Tests.Reader.Sources +{ + public sealed class CachedByteSourceTests + { + private static readonly byte[] Payload = [.. Enumerable.Range(0, 10_000).Select(i => (byte)(i * 7))]; + + [Fact] + public void ReadExactly_Across_Blocks_Returns_The_Inner_Bytes() + { + using CachedByteSource cache = new(new CountingSource(Payload), blockSize: 1024, capacityBytes: 1 << 20); + byte[] read = new byte[3000]; + cache.ReadExactly(500, read); + Assert.Equal(Payload[500..3500], read); + } + + [Fact] + public void Each_Block_Is_Fetched_Once_While_It_Fits() + { + CountingSource inner = new(Payload); + using CachedByteSource cache = new(inner, blockSize: 1024, capacityBytes: 1 << 20); + byte[] chunk = new byte[100]; + for (int pass = 0; pass < 2; pass++) + { + for (int offset = 0; offset < Payload.Length; offset += chunk.Length) + { + cache.ReadExactly(offset, chunk); + } + } + Assert.Equal(10, inner.Reads); + } + + [Fact] + public void Concurrent_Readers_Of_One_Block_Fetch_It_Once() + { + CountingSource inner = new(Payload) { Delay = TimeSpan.FromMilliseconds(50) }; + using CachedByteSource cache = new(inner, blockSize: 4096, capacityBytes: 1 << 20); + Parallel.For(0, 16, _ => + { + byte[] read = new byte[64]; + cache.ReadExactly(0, read); + }); + Assert.Equal(1, inner.Reads); + } + + [Fact] + public void The_Least_Recently_Used_Block_Is_Evicted_Past_Capacity() + { + CountingSource inner = new(Payload); + using CachedByteSource cache = new(inner, blockSize: 1024, capacityBytes: 2048); + byte[] one = new byte[1]; + foreach (long offset in new long[] { 0, 1024, 2048, 1024, 0 }) + { + cache.ReadExactly(offset, one); + } + Assert.Equal(4, inner.Reads); + } + + [Fact] + public void A_Read_At_Or_Past_The_End_Returns_Zero() + { + using CachedByteSource cache = new(new CountingSource(Payload), blockSize: 1024, capacityBytes: 1 << 20); + Assert.Equal(0, cache.Read(Payload.Length, new byte[10])); + Assert.Equal(0, cache.Read(Payload.Length + 5, new byte[10])); + } + + [Fact] + public void A_Failed_Fetch_Is_Tried_Again() + { + CountingSource inner = new(Payload) { FailReads = 1 }; + using CachedByteSource cache = new(inner, blockSize: 1024, capacityBytes: 1 << 20); + byte[] read = new byte[10]; + Assert.Throws(() => cache.ReadExactly(0, read)); + cache.ReadExactly(0, read); + Assert.Equal(Payload[..10], read); + } + + [Fact] + public void Dispose_Disposes_The_Inner_Source_Once() + { + CountingSource inner = new(Payload); + CachedByteSource cache = new(inner, blockSize: 1024, capacityBytes: 1 << 20); + cache.Dispose(); + cache.Dispose(); + Assert.Equal(1, inner.Disposals); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Sources/CountingSource.cs b/tests/ExcelReader.Tests/Reader/Sources/CountingSource.cs new file mode 100644 index 00000000..920e0dc5 --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Sources/CountingSource.cs @@ -0,0 +1,45 @@ +using ExcelReader.Core.Reader.Sources; + +namespace ExcelReader.Tests.Reader.Sources +{ + internal sealed class CountingSource(byte[] bytes) : ByteSource + { + internal int Reads; + internal int Disposals; + internal int FailReads; + internal TimeSpan Delay { get; init; } + + internal override long Length => bytes.Length; + + internal override int Read(long offset, Span destination) + { + ArgumentOutOfRangeException.ThrowIfNegative(offset); + Interlocked.Increment(ref Reads); + if (Delay > TimeSpan.Zero) + { + Thread.Sleep(Delay); + } + if (Interlocked.Decrement(ref FailReads) >= 0) + { + throw new IOException("transient"); + } + if (offset >= bytes.Length) + { + return 0; + } + int count = (int)Math.Min(destination.Length, bytes.Length - offset); + bytes.AsSpan((int)offset, count).CopyTo(destination); + return count; + } + + internal override ValueTask ReadAsync(long offset, Memory destination, CancellationToken ct) + { + return new ValueTask(Read(offset, destination.Span)); + } + + public override void Dispose() + { + Interlocked.Increment(ref Disposals); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Sources/SourceStreamTests.cs b/tests/ExcelReader.Tests/Reader/Sources/SourceStreamTests.cs new file mode 100644 index 00000000..0c381951 --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Sources/SourceStreamTests.cs @@ -0,0 +1,136 @@ +using System.Text; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Sources; +using ExcelReader.Tests.Crypto; + +namespace ExcelReader.Tests.Reader.Sources +{ + public sealed class SourceStreamTests + { + public static TheoryData Formats => new() + { + { ConcurrentSheetTests.Format.Xlsx, ExcelFileFormat.Xlsx }, + { ConcurrentSheetTests.Format.Xlsb, ExcelFileFormat.Xlsb }, + { ConcurrentSheetTests.Format.Xls, ExcelFileFormat.Xls }, + { ConcurrentSheetTests.Format.Xlsx, ExcelFileFormat.Unknown }, + { ConcurrentSheetTests.Format.Xls, ExcelFileFormat.Unknown }, + }; + + public static TheoryData EncryptedNames => new(EncryptedFixtures.All); + + private static List Drain(IExcelSheet sheet) + { + List lines = []; + using IExcelRowEnumerator e = sheet.GetEnumerator(); + while (e.MoveNext()) + { + StringBuilder line = new(); + foreach (RowCell cell in e.Current.Cells) + { + line.Append(cell.Value.GetString()).Append('|'); + } + lines.Add(line.ToString()); + } + return lines; + } + + [Fact] + public void FromStream_Unwraps_A_SourceStream_That_Owns_Its_Source() + { + CountingSource source = new(new byte[100]); + using ByteSource unwrapped = ByteSource.FromStream(new SourceStream(source), leaveOpen: false); + Assert.Same(source, unwrapped); + } + + [Fact] + public void FromStream_Does_Not_Unwrap_A_Stream_The_Caller_Keeps() + { + using SourceStream stream = new(new CountingSource(new byte[100])); + using ByteSource wrapped = ByteSource.FromStream(stream, leaveOpen: true); + Assert.IsType(wrapped); + } + + [Fact] + public void A_Taken_Source_Belongs_To_Its_New_Owner() + { + CountingSource source = new(new byte[100]); + SourceStream stream = new(source); + ByteSource unwrapped = ByteSource.FromStream(stream, leaveOpen: false); + stream.Dispose(); + Assert.Equal(0, source.Disposals); + unwrapped.Dispose(); + Assert.Equal(1, source.Disposals); + } + + [Fact] + public void Disposing_A_Stream_Whose_Source_Was_Not_Taken_Disposes_The_Source() + { + CountingSource source = new(new byte[100]); + new SourceStream(source).Dispose(); + Assert.Equal(1, source.Disposals); + } + + [Fact] + public void Seek_And_Read_Follow_The_Source() + { + byte[] bytes = [.. Enumerable.Range(0, 256).Select(i => (byte)i)]; + using SourceStream stream = new(new CountingSource(bytes)); + Assert.True(stream.CanSeek); + Assert.Equal(256, stream.Length); + stream.Seek(-6, SeekOrigin.End); + byte[] tail = new byte[10]; + Assert.Equal(6, stream.Read(tail)); + Assert.Equal(bytes[250..], tail[..6]); + Assert.Equal(256, stream.Position); + } + + [Theory] + [MemberData(nameof(Formats))] + public void Sheets_Opened_Over_A_SourceStream_Read_In_Parallel_What_Memory_Reads( + ConcurrentSheetTests.Format format, ExcelFileFormat requested) + { + byte[] bytes = ConcurrentSheetTests.Build(format); + List[] expected; + using (IExcelWorkbook reference = Excel.Open(bytes)) + { + expected = [.. Enumerable.Range(0, reference.SheetCount).Select(s => Drain(reference.SheetAt(s)))]; + } + + CountingSource source = new(bytes); + using (IExcelWorkbook workbook = Excel.Open(new SourceStream(source), requested, leaveOpen: false)) + { + List[] parallel = new List[expected.Length * 2]; + Parallel.For(0, parallel.Length, i => parallel[i] = Drain(workbook.SheetAt(i % expected.Length))); + for (int i = 0; i < parallel.Length; i++) + { + Assert.Equal(expected[i % expected.Length], parallel[i]); + } + } + Assert.Equal(1, source.Disposals); + } + + [Fact] + public void A_Csv_Over_A_SourceStream_Reads_More_Than_Once() + { + byte[] bytes = Encoding.UTF8.GetBytes("a,b,c\n1,2,3\n4,5,6\n"); + using IExcelWorkbook csv = Excel.Open(new SourceStream(new CountingSource(bytes)), ExcelFileFormat.Csv, leaveOpen: false); + Assert.Equal(["a|b|c|", "1|2|3|", "4|5|6|"], Drain(csv.FirstSheet)); + Assert.Equal(Drain(csv.FirstSheet), Drain(csv.FirstSheet)); + } + + [Theory] + [MemberData(nameof(EncryptedNames))] + public void An_Encrypted_Package_Over_A_SourceStream_Opens_With_Its_Password(string name) + { + ExcelReaderOptions options = new() { Password = EncryptedFixtures.PasswordFor(name) }; + List expected; + using (IExcelWorkbook plain = Excel.Open(EncryptedFixtures.PlainBytes(name))) + { + expected = Drain(plain.FirstSheet); + } + using IExcelWorkbook workbook = Excel.Open( + new SourceStream(new CountingSource(EncryptedFixtures.Bytes(name))), ExcelFileFormat.Unknown, leaveOpen: false, options); + Assert.Equal(expected, Drain(workbook.FirstSheet)); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Tables/RowIndexTests.cs b/tests/ExcelReader.Tests/Reader/Tables/RowIndexTests.cs new file mode 100644 index 00000000..8e70e9b4 --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Tables/RowIndexTests.cs @@ -0,0 +1,153 @@ +using System.Globalization; +using System.Text; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Internal; +using ExcelReader.Core.Reader.Xlsb; +using ExcelReader.Core.Reader.Xlsx; +using B = ExcelReader.Tests.Reader.Xlsb.Biff12Build; + +namespace ExcelReader.Tests.Reader.Tables +{ + public class RowIndexTests + { + private sealed class StreamBackedMemory(byte[] bytes) : MemoryStream(bytes); + + [Fact] + public void XlsxRowIndexFollowsTheRAttributeAndFallsBackToTheNextRow() + { + using MemoryStream ms = WorkbookBuilder.Build( + """123"""); + using XlsxWorkbook workbook = Excel.FromXlsx(ms); + + Assert.Equal([1, 4, 5, 8], ReadIndexes(workbook.FirstSheet.GetEnumerator())); + } + + [Fact] + public void RowIndexStaysUnsetUntilEnabled() + { + using MemoryStream ms = WorkbookBuilder.Build("""1"""); + using XlsxWorkbook workbook = Excel.FromXlsx(ms); + using XlsxWorkbook.Enumerator rows = workbook.FirstSheet.GetEnumerator(); + + Assert.True(rows.MoveNext()); + Assert.Equal(-1, ((IRowIndexedEnumerator)rows).RowIndex); + } + + [Fact] + public void XlsbRowIndexFollowsTheRowHeaderAndFallsBackToTheNextRow() + { + using MemoryStream ms = XlsbSheet( + [ + .. B.Record(Brt.RowHdr, B.U32(1)), .. B.Record(Brt.CellRk, B.CellRk(0, 0, (1u << 2) | 0x02)), + .. B.Record(Brt.RowHdr, B.U32(4)), .. B.Record(Brt.CellRk, B.CellRk(0, 0, (2u << 2) | 0x02)), + .. B.Record(Brt.RowHdr), .. B.Record(Brt.CellRk, B.CellRk(0, 0, (3u << 2) | 0x02)), + .. B.Record(Brt.EndSheetData), + ]); + using XlsbWorkbook workbook = Excel.FromXlsb(ms); + + Assert.Equal([1, 4, 5], ReadIndexes(workbook.FirstSheet.GetEnumerator())); + } + + [Fact] + public async Task XlsxSyncAndAsyncRowIndexesMatchAcrossBufferRefills() + { + StringBuilder rows = new(); + for (int i = 0; i < 20_000; i++) + { + int r = (3 * i) + 1; + rows.Append(CultureInfo.InvariantCulture, $"""row-{i}-padding-padding-padding"""); + } + using MemoryStream source = WorkbookBuilder.Build(rows.ToString()); + byte[] bytes = source.ToArray(); + int[] expected = [.. Enumerable.Range(0, 20_000).Select(static i => 3 * i)]; + + using XlsxWorkbook syncBook = Excel.FromXlsx(new StreamBackedMemory(bytes), leaveOpen: false); + await using XlsxWorkbook asyncBook = await Excel.FromXlsxAsync( + new StreamBackedMemory(bytes), leaveOpen: false, ct: TestContext.Current.CancellationToken); + + Assert.Equal(expected, ReadIndexes(syncBook.FirstSheet.GetEnumerator())); + Assert.Equal(expected, await ReadIndexesAsync(asyncBook.FirstSheet.GetAsyncEnumerator(TestContext.Current.CancellationToken))); + } + + [Fact] + public async Task XlsbSyncAndAsyncRowIndexesMatchAcrossBufferRefills() + { + List sheet = []; + for (int i = 0; i < 20_000; i++) + { + sheet.AddRange(B.Record(Brt.RowHdr, B.U32((uint)(3 * i)))); + sheet.AddRange(B.Record(Brt.CellSt, B.CellSt(0, 0, $"row-{i}-padding-padding-padding"))); + } + sheet.AddRange(B.Record(Brt.EndSheetData)); + byte[] bytes = XlsbSheet([.. sheet]).ToArray(); + int[] expected = [.. Enumerable.Range(0, 20_000).Select(static i => 3 * i)]; + + using XlsbWorkbook syncBook = Excel.FromXlsb(new StreamBackedMemory(bytes), leaveOpen: false); + await using XlsbWorkbook asyncBook = await Excel.FromXlsbAsync( + new StreamBackedMemory(bytes), leaveOpen: false, ct: TestContext.Current.CancellationToken); + + Assert.Equal(expected, ReadIndexes(syncBook.FirstSheet.GetEnumerator())); + Assert.Equal(expected, await ReadIndexesAsync(asyncBook.FirstSheet.GetAsyncEnumerator(TestContext.Current.CancellationToken))); + } + + [Fact] + public void FixtureRowIndexesSkipAbsentRows() + { + using IExcelWorkbook workbook = Excel.Open(TableMetadataTests.FixturePath("tables.xlsx")); + + Assert.Equal([3, 4, 5, 6, 7, 10], ReadIndexes(workbook.FirstSheet.GetEnumerator())); + } + + private static int[] ReadIndexes(IExcelRowEnumerator rows) + { + using (rows) + { + IRowIndexedEnumerator indexed = (IRowIndexedEnumerator)rows; + indexed.EnableRowIndex(); + List indexes = []; + while (rows.MoveNext()) + { + indexes.Add(indexed.RowIndex); + } + return [.. indexes]; + } + } + + private static async Task ReadIndexesAsync(IExcelRowEnumerator rows) + { + await using (rows.ConfigureAwait(false)) + { + IRowIndexedEnumerator indexed = (IRowIndexedEnumerator)rows; + indexed.EnableRowIndex(); + List indexes = []; + while (await rows.MoveNextAsync().ConfigureAwait(false)) + { + indexes.Add(indexed.RowIndex); + } + return [.. indexes]; + } + } + + private static MemoryStream XlsbSheet(byte[] sheet) + { + MemoryStream ms = new(); + using (var zip = new System.IO.Compression.ZipArchive(ms, System.IO.Compression.ZipArchiveMode.Create, leaveOpen: true)) + { + Add(zip, "xl/workbook.bin", B.Record(Brt.BundleSh, [.. B.U32(0), .. B.U32(0), .. B.WideString("rId1"), .. B.WideString("S1")])); + Add(zip, "xl/_rels/workbook.bin.rels", Encoding.UTF8.GetBytes( + """""")); + Add(zip, "xl/styles.bin", []); + Add(zip, "xl/sharedStrings.bin", []); + Add(zip, "xl/worksheets/sheet1.bin", sheet); + } + ms.Position = 0; + return ms; + } + + private static void Add(System.IO.Compression.ZipArchive zip, string name, byte[] bytes) + { + using Stream stream = zip.CreateEntry(name).Open(); + stream.Write(bytes); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Tables/TableBindingTests.cs b/tests/ExcelReader.Tests/Reader/Tables/TableBindingTests.cs new file mode 100644 index 00000000..ff90b8c3 --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Tables/TableBindingTests.cs @@ -0,0 +1,144 @@ +using ExcelReader.Core.Parser; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Schema; + +namespace ExcelReader.Tests.Reader.Tables +{ + public class TableBindingTests + { + private sealed class SalesRow + { + public string? Product { get; set; } + + public int Qty { get; set; } + + public double Price { get; set; } + } + + private sealed class StockRow + { + public string? Product { get; set; } + + public int Qty { get; set; } + } + + [Theory] + [MemberData(nameof(TableMetadataTests.Fixtures), MemberType = typeof(TableMetadataTests))] + public void SalesBindsItsDataRowsWithoutTheTotalsRow(string file) + { + using IExcelWorkbook workbook = Excel.Open(TableMetadataTests.FixturePath(file)); + + List rows = ExcelParser.FromAttributes().Parse(Table(workbook, "Sales").AsSheet()).ToList(); + + Assert.Equal( + [("Pen", 10, 2.5), ("Notebook", 4, 18.9), ("Pencil", 25, 1.2)], + rows.Select(static r => (r.Product, r.Qty, r.Price))); + } + + [Theory] + [MemberData(nameof(TableMetadataTests.Fixtures), MemberType = typeof(TableMetadataTests))] + public void StockBindsItsOwnColumnsDespiteSharedHeaderNames(string file) + { + using IExcelWorkbook workbook = Excel.Open(TableMetadataTests.FixturePath(file)); + + List rows = ExcelParser.FromAttributes().Parse(Table(workbook, "Stock").AsSheet()).ToList(); + + Assert.Equal([("Pen", 100), ("Notebook", 40), ("Pencil", 250)], rows.Select(static r => (r.Product, r.Qty))); + } + + [Theory] + [MemberData(nameof(TableMetadataTests.Fixtures), MemberType = typeof(TableMetadataTests))] + public async Task StockBindsAsynchronouslyOverAStream(string file) + { + await using IExcelWorkbook workbook = await Excel.OpenAsync( + File.OpenRead(TableMetadataTests.FixturePath(file)), leaveOpen: false, ct: TestContext.Current.CancellationToken); + List rows = []; + + await foreach (StockRow row in ExcelParser.FromAttributes().Parse(Table(workbook, "Stock").AsSheet()) + .WithCancellation(TestContext.Current.CancellationToken)) + { + rows.Add(row); + } + + Assert.Equal([("Pen", 100), ("Notebook", 40), ("Pencil", 250)], rows.Select(static r => (r.Product, r.Qty))); + } + + [Theory] + [MemberData(nameof(TableMetadataTests.Fixtures), MemberType = typeof(TableMetadataTests))] + public void HeaderlessTableRejectsBindingByName(string file) + { + using IExcelWorkbook workbook = Excel.Open(TableMetadataTests.FixturePath(file)); + IExcelSheet view = Table(workbook, "NoHeader").AsSheet(); + + Assert.Throws(() => ExcelParser.FromAttributes().Parse(view)); + Assert.Throws(() => new SheetDataReader(view)); + Assert.Throws(() => Excel.InferSchema(view)); + } + + [Theory] + [MemberData(nameof(TableMetadataTests.Fixtures), MemberType = typeof(TableMetadataTests))] + public void HeaderlessTableReadsWithoutAHeader(string file) + { + using IExcelWorkbook workbook = Excel.Open(TableMetadataTests.FixturePath(file)); + IExcelSheet view = Table(workbook, "NoHeader").AsSheet(); + + using SheetDataReader reader = new(view, new SheetDataReaderOptions { HeaderRow = 0 }); + List letters = []; + while (reader.Read()) + { + letters.Add(reader.GetString(0)); + } + + Assert.Equal(["a", "b", "c"], letters); + Assert.Equal(2, Excel.InferSchema(view, new SchemaInferenceOptions { HeaderRow = 0 }).Count); + } + + [Theory] + [MemberData(nameof(TableMetadataTests.Fixtures), MemberType = typeof(TableMetadataTests))] + public void DataReaderOrdinalsStartAtTheTableFirstColumn(string file) + { + using IExcelWorkbook workbook = Excel.Open(TableMetadataTests.FixturePath(file)); + + using SheetDataReader reader = new(Table(workbook, "Sales").AsSheet()); + Assert.True(reader.Read()); + + Assert.Equal(3, reader.FieldCount); + Assert.Equal(("Product", "Qty", "Price"), (reader.GetName(0), reader.GetName(1), reader.GetName(2))); + Assert.Equal(1, reader.GetOrdinal("Qty")); + Assert.Equal(("Pen", 10.0, 2.5), ((string)reader.GetValue(0), (double)reader.GetValue(1), (double)reader.GetValue(2))); + } + + [Theory] + [MemberData(nameof(TableMetadataTests.Fixtures), MemberType = typeof(TableMetadataTests))] + public void DataReaderOrdinalsLineUpWithOrWithoutAHeaderRow(string file) + { + using IExcelWorkbook workbook = Excel.Open(TableMetadataTests.FixturePath(file)); + IExcelSheet view = Table(workbook, "NoStyle").AsSheet(); + + using SheetDataReader named = new(view, new SheetDataReaderOptions { HeaderRow = 1 }); + Assert.True(named.Read()); + using SheetDataReader unnamed = new(view, new SheetDataReaderOptions { HeaderRow = 0 }); + Assert.True(unnamed.Read()); + + Assert.Equal((2, "Name", "Value", "x"), (named.FieldCount, named.GetName(0), named.GetName(1), named.GetString(0))); + Assert.Equal((2, "Column0", "Name"), (unnamed.FieldCount, unnamed.GetName(0), unnamed.GetString(0))); + } + + [Theory] + [MemberData(nameof(TableMetadataTests.Fixtures), MemberType = typeof(TableMetadataTests))] + public void InferSchemaDropsColumnsLeftOfTheTableAndKeepsAbsoluteIndexes(string file) + { + using IExcelWorkbook workbook = Excel.Open(TableMetadataTests.FixturePath(file)); + + IReadOnlyList schema = Excel.InferSchema(Table(workbook, "Sales").AsSheet()); + + Assert.Equal([("Product", 2), ("Qty", 3), ("Price", 4)], schema.Select(static c => (c.Name, c.Index))); + } + + private static ExcelTable Table(IExcelWorkbook workbook, string name) + { + Assert.True(workbook.TryGetTable(name, out ExcelTable? table)); + return table; + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Tables/TableMetadataTests.cs b/tests/ExcelReader.Tests/Reader/Tables/TableMetadataTests.cs new file mode 100644 index 00000000..00cfca78 --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Tables/TableMetadataTests.cs @@ -0,0 +1,121 @@ +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Xlsx; + +namespace ExcelReader.Tests.Reader.Tables +{ + public class TableMetadataTests + { + private const string Rows = """H1"""; + + public static TheoryData Fixtures { get; } = new() { "tables.xlsx", "tables.xlsb" }; + + private static readonly (string Name, int Sheet, string Ref, int FirstRow, int FirstColumn, int LastRow, int LastColumn, + int HeaderRowCount, int TotalsRowCount, string[] Columns, string? Style)[] Expected = + [ + ("Sales", 0, "C4:E8", 3, 2, 7, 4, 1, 1, ["Product", "Qty", "Price"], "TableStyleMedium2"), + ("Stock", 0, "G4:H7", 3, 6, 6, 7, 1, 0, ["Product", "Qty"], "TableStyleLight9"), + ("NoHeader", 1, "A2:B4", 1, 0, 3, 1, 0, 0, ["Letter", "Number"], "TableStyleMedium2"), + ("NoStyle", 1, "D1:E3", 0, 3, 2, 4, 1, 0, ["Name", "Value"], null), + ]; + + internal static string FixturePath(string file) + { + return Path.Combine(AppContext.BaseDirectory, "data", "tables", file); + } + + [Theory] + [MemberData(nameof(Fixtures))] + public void FixtureTablesReportTheirMetadata(string file) + { + using IExcelWorkbook workbook = Excel.Open(FixturePath(file)); + + AssertExpectedTables(workbook); + } + + [Theory] + [MemberData(nameof(Fixtures))] + public async Task AsyncOpenReportsTheSameTables(string file) + { + await using IExcelWorkbook workbook = await Excel.OpenAsync( + File.OpenRead(FixturePath(file)), leaveOpen: false, ct: TestContext.Current.CancellationToken); + + AssertExpectedTables(workbook); + } + + [Theory] + [MemberData(nameof(Fixtures))] + public void TryGetTableIgnoresCase(string file) + { + using IExcelWorkbook workbook = Excel.Open(FixturePath(file)); + + Assert.True(workbook.TryGetTable("sales", out ExcelTable? sales)); + Assert.Equal("Sales", sales.Name); + Assert.Equal("Data", sales.Sheet.Name); + Assert.False(workbook.TryGetTable("Missing", out _)); + } + + [Fact] + public void EncryptedWorkbookReportsItsTables() + { + using MemoryStream encrypted = new(); + using (FileStream package = File.OpenRead(FixturePath("tables.xlsx"))) + { + Excel.EncryptPackage(package, encrypted, "hunter2"); + } + encrypted.Position = 0; + + using IExcelWorkbook workbook = Excel.Open(encrypted, options: ExcelReaderOptions.Default with { Password = "hunter2" }); + + AssertExpectedTables(workbook); + } + + [Fact] + public void PrefixedTablePartParses() + { + string table = """"""; + using MemoryStream ms = TableWorkbooks.Xlsx(Rows, table); + using XlsxWorkbook workbook = Excel.FromXlsx(ms); + + ExcelTable parsed = Assert.Single(workbook.Tables); + Assert.Equal(("T", "A1:A2", "TableStyleLight1"), (parsed.Name, parsed.Ref, parsed.StyleName)); + Assert.Equal(["H"], parsed.ColumnNames); + } + + [Fact] + public void EscapedColumnNamesAreDecoded() + { + using MemoryStream ms = TableWorkbooks.Xlsx(Rows, TableWorkbooks.Table("A1:A2", """""")); + using XlsxWorkbook workbook = Excel.FromXlsx(ms); + + Assert.Equal(["Qty & Price"], Assert.Single(workbook.Tables).ColumnNames); + } + + [Fact] + public void OnlyTableRelationshipsAreFollowed() + { + string others = + """""" + + """"""; + using MemoryStream ms = TableWorkbooks.Xlsx(Rows, TableWorkbooks.Table("A1:A2", """"""), extraRelationships: others); + using XlsxWorkbook workbook = Excel.FromXlsx(ms); + + Assert.Equal("T", Assert.Single(workbook.Tables).Name); + } + + private static void AssertExpectedTables(IExcelWorkbook workbook) + { + Assert.Equal(Expected.Length, workbook.Tables.Count); + for (int i = 0; i < Expected.Length; i++) + { + ExcelTable table = workbook.Tables[i]; + var expected = Expected[i]; + Assert.Equal((expected.Name, expected.Sheet, expected.Ref), (table.Name, table.Sheet.Index, table.Ref)); + Assert.Equal((expected.FirstRow, expected.FirstColumn, expected.LastRow, expected.LastColumn), + (table.FirstRow, table.FirstColumn, table.LastRow, table.LastColumn)); + Assert.Equal((expected.HeaderRowCount, expected.TotalsRowCount), (table.HeaderRowCount, table.TotalsRowCount)); + Assert.Equal(expected.Columns, table.ColumnNames); + Assert.Equal(expected.Style, table.StyleName); + } + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Tables/TablePartTests.cs b/tests/ExcelReader.Tests/Reader/Tables/TablePartTests.cs new file mode 100644 index 00000000..19d06c16 --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Tables/TablePartTests.cs @@ -0,0 +1,74 @@ +using System.Text; +using ExcelReader.Core.Reader.Internal; + +namespace ExcelReader.Tests.Reader.Tables +{ + public class TablePartTests + { + [Theory] + [InlineData("C4:E8", 3, 2, 7, 4)] + [InlineData("A1", 0, 0, 0, 0)] + [InlineData("AA10:AB12", 9, 26, 11, 27)] + [InlineData("XFD1048576:XFD1048576", 1048575, 16383, 1048575, 16383)] + public void TryParseRangeReadsA1Ranges(string text, int firstRow, int firstColumn, int lastRow, int lastColumn) + { + Assert.True(TablePart.TryParseRange(Encoding.ASCII.GetBytes(text), out int r1, out int c1, out int r2, out int c2)); + Assert.Equal((firstRow, firstColumn, lastRow, lastColumn), (r1, c1, r2, c2)); + } + + [Theory] + [InlineData("")] + [InlineData("C")] + [InlineData("4")] + [InlineData("C0")] + [InlineData("c4")] + [InlineData("C4:")] + [InlineData("C4:E")] + [InlineData("$C$4:$E$8")] + [InlineData("XFE1")] + [InlineData("AAAA1")] + [InlineData("C4x")] + [InlineData("C99999999999")] + public void TryParseRangeRejectsMalformedText(string text) + { + Assert.False(TablePart.TryParseRange(Encoding.ASCII.GetBytes(text), out _, out _, out _, out _)); + } + + [Fact] + public void FormatRangeWritesA1Notation() + { + Assert.Equal("C4:E8", TablePart.FormatRange(3, 2, 7, 4)); + Assert.Equal("XFD1048576:XFD1048576", TablePart.FormatRange(1048575, 16383, 1048575, 16383)); + } + + [Fact] + public void CreateKeepsAWellFormedTable() + { + TablePart part = TablePart.Create(1, 7, "Sales", 3, 2, 7, 4, 1, 1, ["Product", "Qty", "Price"], "TableStyleMedium2"); + + Assert.Equal(("Sales", 1, 7, "TableStyleMedium2"), (part.Name, part.SheetIndex, part.Id, part.StyleName)); + Assert.Equal((3, 2, 7, 4, 1, 1), (part.FirstRow, part.FirstColumn, part.LastRow, part.LastColumn, part.HeaderRowCount, part.TotalsRowCount)); + Assert.Equal(["Product", "Qty", "Price"], part.Columns); + } + + [Theory] + [InlineData("", 0, 0, 1, 0, 1, 0, 1)] + [InlineData("T", -1, 0, 1, 0, 1, 0, 1)] + [InlineData("T", 2, 0, 1, 0, 1, 0, 1)] + [InlineData("T", 0, 0, 1048576, 0, 1, 0, 1)] + [InlineData("T", 0, 1, 1, 0, 1, 0, 1)] + [InlineData("T", 0, 0, 1, 16384, 1, 0, 16385)] + [InlineData("T", 0, 0, 1, 0, 2, 0, 1)] + [InlineData("T", 0, 0, 1, 0, 1, -1, 1)] + [InlineData("T", 0, 0, 0, 0, 1, 1, 1)] + [InlineData("T", 0, 0, 1, 1, 1, 0, 1)] + public void CreateRejectsInconsistentTables(string name, int firstRow, int firstColumn, int lastRow, int lastColumn, + int headerRowCount, int totalsRowCount, int columnCount) + { + string[] columns = [.. Enumerable.Range(0, columnCount).Select(static i => $"C{i}")]; + + Assert.Throws(() => TablePart.Create( + 0, 1, name, firstRow, firstColumn, lastRow, lastColumn, headerRowCount, totalsRowCount, columns, null)); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Tables/TableViewTests.cs b/tests/ExcelReader.Tests/Reader/Tables/TableViewTests.cs new file mode 100644 index 00000000..c8081b55 --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Tables/TableViewTests.cs @@ -0,0 +1,231 @@ +using ExcelReader.Core; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Internal; + +namespace ExcelReader.Tests.Reader.Tables +{ + public class TableViewTests + { + private sealed class FakeRows(int[] rowIndexes) : IExcelRowEnumerator, IRowIndexedEnumerator + { + private int _next; + + public int MoveNextCalls { get; private set; } + + public bool Disposed { get; private set; } + + public bool Enabled { get; private set; } + + public int RowIndex { get; private set; } = -1; + + public Row Current + { + get + { + return default; + } + } + + public void EnableRowIndex() + { + Enabled = true; + } + + public bool MoveNext() + { + MoveNextCalls++; + if (_next == rowIndexes.Length) + { + return false; + } + RowIndex = rowIndexes[_next++]; + return true; + } + + public ValueTask MoveNextAsync() + { + return new ValueTask(MoveNext()); + } + + public void Dispose() + { + Disposed = true; + } + + public ValueTask DisposeAsync() + { + Disposed = true; + return ValueTask.CompletedTask; + } + } + + [Fact] + public void BoundedRowsStopAtTheFirstRowPastTheTable() + { + FakeRows rows = new([0, 3, 4, 5, 6, 7, 8, 9]); + using TableRowEnumerator table = new(rows, firstRow: 3, lastDataRow: 6, firstColumn: 0, lastColumn: 0); + + int yielded = 0; + while (table.MoveNext()) + { + yielded++; + } + + Assert.True(rows.Enabled); + Assert.Equal(4, yielded); + Assert.Equal(6, rows.MoveNextCalls); + Assert.False(table.MoveNext()); + Assert.Equal(6, rows.MoveNextCalls); + } + + [Fact] + public async Task BoundedRowsStopAtTheFirstRowPastTheTableAsync() + { + FakeRows rows = new([0, 3, 4, 5, 6, 7, 8, 9]); + await using TableRowEnumerator table = new(rows, firstRow: 3, lastDataRow: 6, firstColumn: 0, lastColumn: 0); + + int yielded = 0; + while (await table.MoveNextAsync()) + { + yielded++; + } + + Assert.Equal(4, yielded); + Assert.Equal(6, rows.MoveNextCalls); + } + + public static TheoryData MalformedRowIndexes { get; } = new() + { + new[] { 0, -1 }, + new[] { 0, ExcelLimits.MaxRows }, + new[] { 0, 2, 2 }, + new[] { 0, 3, 2 }, + }; + + [Theory] + [MemberData(nameof(MalformedRowIndexes))] + public void MalformedRowIndexThrowsInvalidData(int[] rowIndexes) + { + using TableRowEnumerator table = new(new FakeRows(rowIndexes), firstRow: 0, lastDataRow: 10, firstColumn: 0, lastColumn: 0); + + Assert.Throws(() => + { + while (table.MoveNext()) + { + } + }); + } + + [Theory] + [MemberData(nameof(MalformedRowIndexes))] + public async Task MalformedRowIndexThrowsInvalidDataAsync(int[] rowIndexes) + { + await using TableRowEnumerator table = new(new FakeRows(rowIndexes), firstRow: 0, lastDataRow: 10, firstColumn: 0, lastColumn: 0); + + await Assert.ThrowsAsync(async () => + { + while (await table.MoveNextAsync()) + { + } + }); + } + + [Fact] + public void DisposingTheViewDisposesTheSheetEnumerator() + { + FakeRows rows = new([0]); + TableRowEnumerator table = new(rows, firstRow: 0, lastDataRow: 0, firstColumn: 0, lastColumn: 0); + + table.Dispose(); + + Assert.True(rows.Disposed); + } + + [Theory] + [MemberData(nameof(TableMetadataTests.Fixtures), MemberType = typeof(TableMetadataTests))] + public void SalesViewYieldsHeaderAndDataRowsCutToTheTable(string file) + { + using IExcelWorkbook workbook = Excel.Open(TableMetadataTests.FixturePath(file)); + + Assert.Equal( + ["2,3,4:Product,Qty", "2,3,4:Pen,10", "2,3,4:Notebook,4", "2,3,4:Pencil,25"], + Render(Table(workbook, "Sales").AsSheet(), 2, 3)); + } + + [Theory] + [MemberData(nameof(TableMetadataTests.Fixtures), MemberType = typeof(TableMetadataTests))] + public void StockViewIgnoresTheNeighbouringTableAndCells(string file) + { + using IExcelWorkbook workbook = Excel.Open(TableMetadataTests.FixturePath(file)); + + Assert.Equal( + ["6,7:Product,Qty", "6,7:Pen,100", "6,7:Notebook,40", "6,7:Pencil,250"], + Render(Table(workbook, "Stock").AsSheet(), 6, 7)); + } + + [Theory] + [MemberData(nameof(TableMetadataTests.Fixtures), MemberType = typeof(TableMetadataTests))] + public void HeaderlessViewYieldsOnlyDataRows(string file) + { + using IExcelWorkbook workbook = Excel.Open(TableMetadataTests.FixturePath(file)); + + Assert.Equal(["0,1:a,1", "0,1:b,2", "0,1:c,3"], Render(Table(workbook, "NoHeader").AsSheet(), 0, 1)); + } + + [Theory] + [MemberData(nameof(TableMetadataTests.Fixtures), MemberType = typeof(TableMetadataTests))] + public void ViewReportsItsOwningSheet(string file) + { + using IExcelWorkbook workbook = Excel.Open(TableMetadataTests.FixturePath(file)); + IExcelSheet view = Table(workbook, "NoStyle").AsSheet(); + + Assert.Equal(("Extra", 1, ExcelSheetVisibility.Visible), (view.Name, view.Index, view.Visibility)); + Assert.Equal(["3,4:Name,Value", "3,4:x,1", "3,4:y,2"], Render(view, 3, 4)); + } + + [Theory] + [MemberData(nameof(TableMetadataTests.Fixtures), MemberType = typeof(TableMetadataTests))] + public async Task AsyncViewOverAStreamYieldsTheSameRows(string file) + { + await using IExcelWorkbook workbook = await Excel.OpenAsync( + File.OpenRead(TableMetadataTests.FixturePath(file)), leaveOpen: false, ct: TestContext.Current.CancellationToken); + IExcelSheet view = Table(workbook, "Sales").AsSheet(); + + List rows = []; + await using IExcelRowEnumerator enumerator = view.GetAsyncEnumerator(TestContext.Current.CancellationToken); + while (await enumerator.MoveNextAsync()) + { + rows.Add(RenderRow(enumerator.Current, 2, 3)); + } + + Assert.Equal(["2,3,4:Product,Qty", "2,3,4:Pen,10", "2,3,4:Notebook,4", "2,3,4:Pencil,25"], rows); + } + + private static ExcelTable Table(IExcelWorkbook workbook, string name) + { + Assert.True(workbook.TryGetTable(name, out ExcelTable? table)); + return table; + } + + private static List Render(IExcelSheet sheet, int firstShown, int secondShown) + { + List rows = []; + using IExcelRowEnumerator enumerator = sheet.GetEnumerator(); + while (enumerator.MoveNext()) + { + rows.Add(RenderRow(enumerator.Current, firstShown, secondShown)); + } + return rows; + } + + private static string RenderRow(Row row, int firstShown, int secondShown) + { + List columns = []; + foreach (RowCell cell in row.Cells) + { + columns.Add(cell.ColumnIndex); + } + return $"{string.Join(',', columns)}:{row[firstShown].GetString()},{row[secondShown].GetString()}"; + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Tables/TableWorkbooks.cs b/tests/ExcelReader.Tests/Reader/Tables/TableWorkbooks.cs new file mode 100644 index 00000000..5ae6543c --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Tables/TableWorkbooks.cs @@ -0,0 +1,91 @@ +using System.IO.Compression; +using System.Text; +using ExcelReader.Core.Reader.Xlsb; +using B = ExcelReader.Tests.Reader.Xlsb.Biff12Build; + +namespace ExcelReader.Tests.Reader.Tables +{ + internal static class TableWorkbooks + { + internal const string TableRelationship = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/table"; + private const string PackageRelationships = "http://schemas.openxmlformats.org/package/2006/relationships"; + private const string Main = "http://schemas.openxmlformats.org/spreadsheetml/2006/main"; + + internal static MemoryStream Xlsx(string rows, string tableXml, string target = "../tables/table1.xml", string extraRelationships = "") + { + string rels = $"""{extraRelationships}"""; + return WorkbookBuilder.BuildMultiSheet( + [("S1", rows)], + extraParts: [("xl/worksheets/_rels/sheet1.xml.rels", rels), ("xl/tables/table1.xml", tableXml)]); + } + + internal static string Table(string reference, string columns, string attributes = "") + { + return $"""{columns}
"""; + } + + internal static MemoryStream Xlsb(byte[] tableBin, byte[]? sheetBin = null) + { + MemoryStream ms = new(); + using (var zip = new ZipArchive(ms, ZipArchiveMode.Create, leaveOpen: true)) + { + Add(zip, "xl/workbook.bin", B.Record(Brt.BundleSh, [.. B.U32(0), .. B.U32(0), .. B.WideString("rId1"), .. B.WideString("S1")])); + Add(zip, "xl/_rels/workbook.bin.rels", Encoding.UTF8.GetBytes( + """""")); + Add(zip, "xl/styles.bin", []); + Add(zip, "xl/sharedStrings.bin", []); + Add(zip, "xl/worksheets/sheet1.bin", sheetBin ?? + [ + .. B.Record(Brt.RowHdr, B.U32(0)), + .. B.Record(Brt.CellSt, B.CellSt(0, 0, "H")), + .. B.Record(Brt.RowHdr, B.U32(1)), + .. B.Record(Brt.CellRk, B.CellRk(0, 0, (1u << 2) | 0x02)), + .. B.Record(Brt.EndSheetData), + ]); + Add(zip, "xl/worksheets/_rels/sheet1.bin.rels", Encoding.UTF8.GetBytes( + $"""""")); + Add(zip, "xl/tables/table1.bin", tableBin); + } + ms.Position = 0; + return ms; + } + + internal static byte[] BeginList(uint firstRow, uint lastRow, uint firstColumn, uint lastColumn, uint headerRowCount, uint totalsRowCount, string name) + { + return B.Record(Brt.BeginList, + [ + .. B.U32(firstRow), .. B.U32(lastRow), .. B.U32(firstColumn), .. B.U32(lastColumn), + .. B.U32(0), .. B.U32(1), .. B.U32(headerRowCount), .. B.U32(totalsRowCount), .. B.U32(0), + .. Ones(24), .. B.U32(0), + .. B.WideString(name), .. B.WideString(name), .. B.WideString(""), + .. B.NullWideString(), .. B.NullWideString(), .. B.NullWideString(), + ]); + } + + internal static byte[] ListColumn(uint id, string caption) + { + return B.Record(Brt.BeginListCol, + [ + .. B.U32(id), .. B.U32(0), .. Ones(12), .. B.U32(0), + .. B.NullWideString(), .. B.WideString(caption), + .. B.NullWideString(), .. B.NullWideString(), .. B.NullWideString(), .. B.NullWideString(), + ]); + } + + internal static byte[] StyleClient(string style) + { + return B.Record(Brt.ListTableStyleClient, [.. B.U16(4), .. B.WideString(style)]); + } + + private static byte[] Ones(int count) + { + return Enumerable.Repeat((byte)0xFF, count).ToArray(); + } + + private static void Add(ZipArchive zip, string name, byte[] bytes) + { + using Stream stream = zip.CreateEntry(name).Open(); + stream.Write(bytes); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Tables/WorkbookTablesTests.cs b/tests/ExcelReader.Tests/Reader/Tables/WorkbookTablesTests.cs new file mode 100644 index 00000000..572a5106 --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Tables/WorkbookTablesTests.cs @@ -0,0 +1,51 @@ +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Csv; +using ExcelReader.Core.Reader.Xls; +using ExcelReader.Core.Reader.Xlsx; +using ExcelReader.Tests.Reader.Xls; + +namespace ExcelReader.Tests.Reader.Tables +{ + public class WorkbookTablesTests + { + [Fact] + public void XlsxWithoutTablesReportsNone() + { + using MemoryStream ms = WorkbookBuilder.Build("""1"""); + using XlsxWorkbook workbook = Excel.FromXlsx(ms); + + Assert.Empty(workbook.Tables); + Assert.False(workbook.TryGetTable("Sales", out _)); + } + + [Fact] + public void XlsReportsNoTables() + { + using MemoryStream ms = XlsWorkbookBuilder.Build(false, null, null, ("S1", [["a"]])); + using XlsWorkbook workbook = Excel.FromXls(ms); + + Assert.Empty(workbook.Tables); + Assert.False(workbook.TryGetTable("Sales", out _)); + } + + [Fact] + public void CsvReportsNoTables() + { + using CsvReader workbook = Excel.FromCsv("a,b\n1,2\n"u8.ToArray()); + + Assert.Empty(workbook.Tables); + Assert.False(workbook.TryGetTable("Sales", out _)); + } + + [Fact] + public void TablesThrowOnceTheWorkbookIsDisposed() + { + using MemoryStream ms = WorkbookBuilder.Build("""1"""); + XlsxWorkbook workbook = Excel.FromXlsx(ms); + workbook.Dispose(); + + Assert.Throws(() => workbook.Tables); + Assert.Throws(() => workbook.TryGetTable("Sales", out _)); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Utf8StringCacheTests.cs b/tests/ExcelReader.Tests/Reader/Utf8StringCacheTests.cs index cf354554..6ed9cf32 100644 --- a/tests/ExcelReader.Tests/Reader/Utf8StringCacheTests.cs +++ b/tests/ExcelReader.Tests/Reader/Utf8StringCacheTests.cs @@ -95,9 +95,8 @@ public void CsvInternStringsDefaultsToFalse() [Fact] public void CsvInternStringsTrueEnablesDedupAcrossRows() { - var options = new CsvReaderOptions { InternStrings = true }; using MemoryStream ms = new(Encoding.UTF8.GetBytes("alpha,1\nalpha,2\nbeta,3\n")); - using CsvReader reader = Excel.FromCsv(ms, options: options); + using CsvReader reader = Excel.FromCsv(ms, options: new ExcelReaderOptions { InternStrings = true }); using CsvReader.Enumerator e = reader.FirstSheet.GetEnumerator(); Assert.True(e.MoveNext()); diff --git a/tests/ExcelReader.Tests/Reader/Xls/XlsWorkbookBuilder.cs b/tests/ExcelReader.Tests/Reader/Xls/XlsWorkbookBuilder.cs index 5230335f..04dd0027 100644 --- a/tests/ExcelReader.Tests/Reader/Xls/XlsWorkbookBuilder.cs +++ b/tests/ExcelReader.Tests/Reader/Xls/XlsWorkbookBuilder.cs @@ -12,6 +12,7 @@ internal sealed record XlsDate(DateTime Value); internal sealed record XlsError(byte Code); internal sealed record XlsFormula(double Value); internal sealed record XlsFormulaBool(bool Value); + internal sealed record XlsFormulaDate(DateTime Value); internal sealed record XlsFormulaError(byte Code); internal sealed record XlsRkInt(int Value); internal sealed record XlsRkRaw(uint Value); @@ -319,6 +320,9 @@ private static void WriteCell( byte[] data = [.. U16(row), .. U16(col), .. U16(0), .. Double(f.Value), .. new byte[8]]; WriteRecord(sheet, 0x0006, data); break; + case XlsFormulaDate f: + WriteRecord(sheet, 0x0006, [.. U16(row), .. U16(col), .. U16(1), .. Double(f.Value.ToOADate()), .. new byte[8]]); + break; case XlsFormulaBool f: WriteRecord(sheet, 0x0006, [.. U16(row), .. U16(col), .. U16(0), .. FormulaSpecial(1, (byte)(f.Value ? 1 : 0)), .. new byte[8]]); break; diff --git a/tests/ExcelReader.Tests/Reader/Xls/XlsWorkbookTests.cs b/tests/ExcelReader.Tests/Reader/Xls/XlsWorkbookTests.cs index aff305a6..514e1d66 100644 --- a/tests/ExcelReader.Tests/Reader/Xls/XlsWorkbookTests.cs +++ b/tests/ExcelReader.Tests/Reader/Xls/XlsWorkbookTests.cs @@ -38,11 +38,25 @@ public void ReadsBasicBiff8Cells() Assert.Equal("1", row[2].GetString()); Assert.Equal(CellType.Error, row[3].Type); Assert.Equal("#DIV/0!", row[3].GetString()); - Assert.Equal(CellType.Formula, row[4].Type); + Assert.Equal(CellType.Number, row[4].Type); Assert.Equal("12.5", row[4].GetString()); Assert.False(e.MoveNext()); } + [Fact] + public void AFormulaWithADateStyleReadsAsADate() + { + var date = new DateTime(2024, 5, 6, 0, 0, 0, DateTimeKind.Unspecified); + using var ms = XlsWorkbookBuilder.Build(sheets: [("S1", [[new XlsFormulaDate(date)]])]); + using var reader = Excel.FromXls(ms); + + using var e = reader.FirstSheet.GetEnumerator(); + Assert.True(e.MoveNext()); + Assert.Equal(CellType.Date, e.Current[0].Type); + Assert.True(e.Current[0].TryGetDateTime(isDate1904: false, out DateTime parsed)); + Assert.Equal(date, parsed); + } + [Fact] public void ReadsAdditionalBiff8CellEncodings() { @@ -211,7 +225,7 @@ public void CustomDateFormatMarksDateCells() Assert.True(e.MoveNext()); Assert.True(e.MoveNext()); Assert.Equal(CellType.Date, e.Current[0].Type); - Assert.True(e.Current[0].TryGetDateTime(out DateTime parsed)); + Assert.True(e.Current[0].TryGetDateTime(isDate1904: false, out DateTime parsed)); Assert.Equal(date, parsed); } @@ -305,7 +319,7 @@ public void XlsParserHeaderRowSkipsLeadingRows() { using var ms = XlsWorkbookBuilder.Build(sheets: [("S1", [["skip"], ["Name"], ["Sol"]])]); using var reader = Excel.FromXls(ms); - var parser = ExcelParser.FromAttributes(new ExcelParserConfig { HeaderRow = 2 }); + var parser = ExcelParser.FromAttributes(new ExcelParserOptions { HeaderRow = 2 }); var result = parser.Parse(reader.FirstSheet).ToList(); Assert.Single(result); diff --git a/tests/ExcelReader.Tests/Reader/Xlsx/CorruptSheetEntryTests.cs b/tests/ExcelReader.Tests/Reader/Xlsx/CorruptSheetEntryTests.cs new file mode 100644 index 00000000..89f8c156 --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Xlsx/CorruptSheetEntryTests.cs @@ -0,0 +1,46 @@ +using System.Globalization; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Xlsx; + +namespace ExcelReader.Tests.Reader.Xlsx +{ + public class CorruptSheetEntryTests + { + private static async Task BuildAsync(int rows) + { + object?[][] data = new object?[rows + 1][]; + data[0] = ["Name", "Qty", "Day"]; + for (int i = 0; i < rows; i++) + { + data[i + 1] = [string.Create(CultureInfo.InvariantCulture, $"item{i:D6}"), i, new DateTime(2024, 1, 1).AddDays(i % 400)]; + } + using MemoryStream ms = await TypedWorkbook.BuildAsync(data); + return ms.ToArray(); + } + + private static int CountRows(XlsxWorkbook workbook) + { + int count = 0; + foreach (Row _ in workbook.FirstSheet) + { + count++; + } + return count; + } + + // Damage at +4096 decodes as a stream that ends early: before the fix every reader stopped at 298 rows without an error. + [Theory] + [InlineData(4096, false)] + [InlineData(4096, true)] + [InlineData(300_000, false)] + public async Task ADamagedSheetEntryThrowsInsteadOfEndingEarly(int offset, bool prefetch) + { + byte[] bytes = await BuildAsync(30_000); + int entry = bytes.AsSpan().IndexOf("xl/worksheets/sheet1.xml"u8); + Array.Fill(bytes, (byte)0xFF, entry + offset, 512); + using XlsxWorkbook workbook = Excel.FromXlsx(bytes, new ExcelReaderOptions { PrefetchDecompression = prefetch }); + + Assert.Throws(() => CountRows(workbook)); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Xlsx/RealWorldXlsxCorpusTests.cs b/tests/ExcelReader.Tests/Reader/Xlsx/RealWorldXlsxCorpusTests.cs index 0516f401..ccb3324f 100644 --- a/tests/ExcelReader.Tests/Reader/Xlsx/RealWorldXlsxCorpusTests.cs +++ b/tests/ExcelReader.Tests/Reader/Xlsx/RealWorldXlsxCorpusTests.cs @@ -19,7 +19,7 @@ public void ReadsSheetJsGeneratedWorkbook() Assert.Equal("price", e.Current[2].GetString()); Assert.Equal("in_stock", e.Current[3].GetString()); Assert.Equal("restock_date", e.Current[4].GetString()); - Assert.Equal(CellType.Formula, e.Current[0].Type); + Assert.Equal(CellType.ExcelString, e.Current[0].Type); Assert.True(e.MoveNext()); Assert.Equal("Widget", e.Current[0].GetString()); @@ -30,19 +30,19 @@ public void ReadsSheetJsGeneratedWorkbook() Assert.Equal(CellType.Boolean, e.Current[3].Type); Assert.Equal("1", e.Current[3].GetString()); Assert.Equal(CellType.Date, e.Current[4].Type); - Assert.True(e.Current[4].TryGetDateTime(out DateTime restock)); + Assert.True(e.Current[4].TryGetDateTime(isDate1904: false, out DateTime restock)); Assert.Equal(new DateTime(2024, 1, 14, 21, 0, 0, DateTimeKind.Unspecified), restock); Assert.True(e.MoveNext()); Assert.Equal("Gadget", e.Current[0].GetString()); Assert.True(e.Current[1].TryParse(null, out int gadgetQty)); Assert.Equal(0, gadgetQty); - Assert.True(e.Current[4].TryGetDateTime(out DateTime gadgetDate)); + Assert.True(e.Current[4].TryGetDateTime(isDate1904: false, out DateTime gadgetDate)); Assert.Equal(new DateTime(2024, 2, 29, 21, 0, 0, DateTimeKind.Unspecified), gadgetDate); Assert.True(e.MoveNext()); Assert.Equal("Gizmo", e.Current[0].GetString()); - Assert.True(e.Current[4].TryGetDateTime(out DateTime gizmoDate)); + Assert.True(e.Current[4].TryGetDateTime(isDate1904: false, out DateTime gizmoDate)); Assert.Equal(new DateTime(2024, 6, 29, 21, 0, 0, DateTimeKind.Unspecified), gizmoDate); Assert.False(e.MoveNext()); diff --git a/tests/ExcelReader.Tests/Reader/Xlsx/XStringEscapeTests.cs b/tests/ExcelReader.Tests/Reader/Xlsx/XStringEscapeTests.cs new file mode 100644 index 00000000..0d56ae59 --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Xlsx/XStringEscapeTests.cs @@ -0,0 +1,123 @@ +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Xlsx; +using ExcelReader.Core.Writer.Xlsx; +using ExcelReader.Tests.Writer.Tables; + +namespace ExcelReader.Tests.Reader.Xlsx +{ + public class XStringEscapeTests + { + public static TheoryData RoundTripText() + { + return new() + { + "a\rb", + "line1\r\nline2", + "a\u0001b _x0041_", + "_x000D_", + "x_y_z", + "trailing_", + "_x0041", + "\u001F\u0000", + }; + } + + private static List ReadFirstColumn(Stream workbook) + { + using XlsxWorkbook reader = Excel.FromXlsx(workbook); + var values = new List(); + foreach (Row row in reader.FirstSheet) + { + values.Add(row[0].GetString()); + } + return values; + } + + [Theory] + [MemberData(nameof(RoundTripText))] + public async Task TextWrittenByTheLibraryReadsBackUnchanged(string text) + { + foreach (bool shared in new[] { false, true }) + { + var ms = new MemoryStream(); + await using (XlsxWorkbookWriter wb = XlsxWorkbookWriter.Create(ms, leaveOpen: true, new XlsxWriterOptions { UseSharedStrings = shared })) + { + XlsxSheetWriter sheet = wb.AddSheet("S1"); + await using (XlsxRowWriter row = await sheet.StartRowAsync(TestContext.Current.CancellationToken)) + { + row.Write(text); + } + await sheet.EndAsync(TestContext.Current.CancellationToken); + await wb.EndAsync(TestContext.Current.CancellationToken); + } + ms.Position = 0; + + Assert.Equal([text], ReadFirstColumn(ms)); + } + } + + public static TheoryData ExcelEscapes() + { + return new() + { + { "a_x000D_b", "a\rb" }, + { "a_x000d_b", "a\rb" }, + { "_x000D__x000A_", "\r\n" }, + { "_x005F_x0041_", "_x0041_" }, + { "_x0041_", "A" }, + { "_xD83D__xDE00_", "\U0001F600" }, + { "_xD800_", "_xD800_" }, + { "_xDE00_x", "_xDE00_x" }, + { "_X000D_", "_X000D_" }, + { "_x00D_", "_x00D_" }, + { "_x000G_", "_x000G_" }, + { "_x00!0_", "_x00!0_" }, + { "_x00@0_", "_x00@0_" }, + { "a_x000D", "a_x000D" }, + { "&_x0026_", "&&" }, + }; + } + + [Theory] + [MemberData(nameof(ExcelEscapes))] + public void EscapesInSharedInlineAndFormulaTextAreDecoded(string xml, string expected) + { + string rows = + $"""0{xml}A1{xml}1"""; + string sharedStrings = $"""{xml}[{xml}]"""; + using MemoryStream ms = WorkbookBuilder.Build(rows, sharedStrings); + using XlsxWorkbook reader = Excel.FromXlsx(ms); + + using XlsxWorkbook.Enumerator e = reader.FirstSheet.GetEnumerator(); + Assert.True(e.MoveNext()); + Row row = e.Current; + Assert.Equal(expected, row[0].GetString()); + Assert.Equal(expected, row[1].GetString()); + Assert.Equal(expected, row[2].GetString()); + Assert.Equal("[" + expected + "]", row[3].GetString()); + } + + [Fact] + public void APrefixedSheetDecodesEscapesInInlineText() + { + using MemoryStream ms = WorkbookBuilder.BuildPrefixed( + "x", """a_x000D_b"""); + + Assert.Equal(["a\rb"], ReadFirstColumn(ms)); + } + + [Fact] + public void ATableColumnNameWrittenByTheLibraryReadsBackUnchanged() + { + ITableBook book = TableBooks.Create("xlsx"); + book.AddSheet("S"); + book.BeginTable("Codes", ["Code_x0041_", "Line_x000D_"]); + book.Row("a", "b"); + byte[] package = book.Finish(); + + using XlsxWorkbook reader = Excel.FromXlsx(package); + Assert.True(reader.TryGetTable("Codes", out ExcelTable? table)); + Assert.Equal(["Code_x0041_", "Line_x000D_"], table.ColumnNames); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Xlsx/XlsxAsyncRowBoundaryTests.cs b/tests/ExcelReader.Tests/Reader/Xlsx/XlsxAsyncRowBoundaryTests.cs new file mode 100644 index 00000000..09a78e19 --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Xlsx/XlsxAsyncRowBoundaryTests.cs @@ -0,0 +1,175 @@ +using System.Globalization; +using System.IO.Compression; +using System.Text; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Xlsx; + +namespace ExcelReader.Tests.Reader.Xlsx +{ + public class XlsxAsyncRowBoundaryTests + { + private const int RowCount = 4_000; + + private sealed class YieldingStream(byte[] bytes) : Stream + { + private const int MaxPerRead = 4096; + + private long _position; + + public override bool CanRead + { + get + { + return true; + } + } + + public override bool CanSeek + { + get + { + return true; + } + } + + public override bool CanWrite + { + get + { + return false; + } + } + + public override long Length + { + get + { + return bytes.Length; + } + } + + public override long Position + { + get + { + return _position; + } + set + { + _position = value; + } + } + + public override int Read(byte[] buffer, int offset, int count) + { + return Read(buffer.AsSpan(offset, count)); + } + + public override int Read(Span buffer) + { + int available = (int)Math.Max(0, Math.Min(bytes.Length - _position, Math.Min(buffer.Length, MaxPerRead))); + bytes.AsSpan((int)_position, available).CopyTo(buffer); + _position += available; + return available; + } + + public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + await Task.Yield(); + return Read(buffer.Span); + } + + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + return ReadAsync(buffer.AsMemory(offset, count), cancellationToken).AsTask(); + } + + public override long Seek(long offset, SeekOrigin origin) + { + _position = origin switch + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => _position + offset, + _ => bytes.Length + offset, + }; + return _position; + } + + public override void Flush() + { + } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotSupportedException(); + } + } + + [Fact] + public async Task AsyncReadKeepsRowsWhoseOpenTagStraddlesARefill() + { + byte[] bytes = BuildWideRowTagWorkbook(); + string[] expected = [.. Enumerable.Range(1, RowCount).Select(static i => i.ToString(CultureInfo.InvariantCulture))]; + + await using XlsxWorkbook workbook = await Excel.FromXlsxAsync( + new YieldingStream(bytes), leaveOpen: false, ct: TestContext.Current.CancellationToken); + List values = []; + await using (XlsxWorkbook.Enumerator rows = workbook.FirstSheet.GetAsyncEnumerator(TestContext.Current.CancellationToken)) + { + while (await rows.MoveNextAsync()) + { + values.Add(rows.Current[0].GetString()); + } + } + + Assert.Equal(expected, values, StringComparer.Ordinal); + Assert.Equal(expected, ReadSync(bytes), StringComparer.Ordinal); + } + + private static List ReadSync(byte[] bytes) + { + using XlsxWorkbook workbook = Excel.FromXlsx(new YieldingStream(bytes), leaveOpen: false); + using XlsxWorkbook.Enumerator rows = workbook.FirstSheet.GetEnumerator(); + List values = []; + while (rows.MoveNext()) + { + values.Add(rows.Current[0].GetString()); + } + return values; + } + + private static byte[] BuildWideRowTagWorkbook() + { + string padding = new('p', 400); + StringBuilder rows = new(); + for (int i = 1; i <= RowCount; i++) + { + rows.Append(CultureInfo.InvariantCulture, + $"""{i}"""); + } + using MemoryStream compressed = WorkbookBuilder.Build(rows.ToString()); + return Stored(compressed); + } + + private static byte[] Stored(MemoryStream compressed) + { + using MemoryStream stored = new(); + using (ZipArchive source = new(compressed, ZipArchiveMode.Read)) + using (ZipArchive target = new(stored, ZipArchiveMode.Create, leaveOpen: true)) + { + foreach (ZipArchiveEntry entry in source.Entries) + { + using Stream from = entry.Open(); + using Stream to = target.CreateEntry(entry.FullName, CompressionLevel.NoCompression).Open(); + from.CopyTo(to); + } + } + return stored.ToArray(); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Xlsx/XlsxDialectShapeTests.cs b/tests/ExcelReader.Tests/Reader/Xlsx/XlsxDialectShapeTests.cs index 25ca20e9..abe8e018 100644 --- a/tests/ExcelReader.Tests/Reader/Xlsx/XlsxDialectShapeTests.cs +++ b/tests/ExcelReader.Tests/Reader/Xlsx/XlsxDialectShapeTests.cs @@ -73,7 +73,7 @@ private static IEnumerable LibraryAndCloudFixtures() """, null, [ - new ExpectedCell(0, 0, "ab", CellType.Formula), + new ExpectedCell(0, 0, "ab", CellType.ExcelString), new ExpectedCell(0, 1, "#DIV/0!", CellType.Error), new ExpectedCell(0, 2, "0", CellType.Boolean), ]) diff --git a/tests/ExcelReader.Tests/Reader/Xlsx/XlsxProducerDialectShapeTests.cs b/tests/ExcelReader.Tests/Reader/Xlsx/XlsxProducerDialectShapeTests.cs index 566dfe0b..6aee085d 100644 --- a/tests/ExcelReader.Tests/Reader/Xlsx/XlsxProducerDialectShapeTests.cs +++ b/tests/ExcelReader.Tests/Reader/Xlsx/XlsxProducerDialectShapeTests.cs @@ -72,7 +72,7 @@ private static IEnumerable DotNetAndJsProducerFixtures() [ new ExpectedCell(0, 0, CellType.Date, "45292"), new ExpectedCell(0, 1, CellType.Empty, ""), - new ExpectedCell(0, 2, CellType.Formula, "AB"), + new ExpectedCell(0, 2, CellType.ExcelString, "AB"), new ExpectedCell(0, 3, CellType.Empty, ""), new ExpectedCell(0, 4, CellType.Error, "#N/A"), ]) diff --git a/tests/ExcelReader.Tests/Reader/Zip/HuffmanTableBuilderTests.cs b/tests/ExcelReader.Tests/Reader/Zip/HuffmanTableBuilderTests.cs new file mode 100644 index 00000000..27365b70 --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Zip/HuffmanTableBuilderTests.cs @@ -0,0 +1,161 @@ +using ExcelReader.Core.Reader.Zip.Inflate; + +namespace ExcelReader.Tests.Reader.Zip +{ + public class HuffmanTableBuilderTests + { + private static readonly uint[] _symbolResults = [.. Enumerable.Range(0, InflateFormat.MaxLitlenSymbols).Select(static i => (uint)i << 16)]; + + public static TheoryData CompleteCodes() + { + byte[] fixedLitlen = new byte[288]; + fixedLitlen.AsSpan(0, 144).Fill(8); + fixedLitlen.AsSpan(144, 112).Fill(9); + fixedLitlen.AsSpan(256, 24).Fill(7); + fixedLitlen.AsSpan(280, 8).Fill(8); + return new TheoryData + { + { [3, 3, 3, 3, 3, 2, 4, 4], 11 }, + { [3, 3, 3, 3, 3, 2, 4, 4], 3 }, + { fixedLitlen, 11 }, + { fixedLitlen, 8 }, + { [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 15], 11 }, + { [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 15], 8 }, + { [0, 2, 0, 2, 2, 0, 0, 2], 8 }, + }; + } + + [Theory] + [MemberData(nameof(CompleteCodes))] + public void EveryCanonicalCodewordDecodesToItsSymbol(byte[] lens, int tableBits) + { + uint[] table = new uint[InflateFormat.LitlenTableSize]; + ushort[] sorted = new ushort[lens.Length]; + + Assert.True(HuffmanTableBuilder.TryBuild(table, lens, _symbolResults, tableBits, 15, sorted, shrink: false, out int used)); + Assert.Equal(tableBits, used); + + uint[] codewords = CanonicalCodewords(lens); + int checkedSymbols = 0; + for (int symbol = 0; symbol < lens.Length; symbol++) + { + if (lens[symbol] == 0) + { + continue; + } + uint reversed = Reverse(codewords[symbol], lens[symbol]); + foreach (uint padding in (uint[])[0, uint.MaxValue]) + { + uint bits = reversed | (padding << lens[symbol]); + (int decoded, int length) = Lookup(table, bits, tableBits); + Assert.Equal(symbol, decoded); + Assert.Equal(lens[symbol], length); + } + checkedSymbols++; + } + Assert.Equal(lens.Count(static len => len != 0), checkedSymbols); + } + + [Fact] + public void ShrinkNarrowsTheTableToTheLongestCodeword() + { + uint[] table = new uint[InflateFormat.LitlenTableSize]; + ushort[] sorted = new ushort[8]; + + Assert.True(HuffmanTableBuilder.TryBuild(table, [3, 3, 3, 3, 3, 2, 4, 4], _symbolResults, 11, 15, sorted, shrink: true, out int used)); + + Assert.Equal(4, used); + } + + [Theory] + [InlineData(new byte[] { 1, 1, 1 })] + [InlineData(new byte[] { 2, 2, 2, 2, 2 })] + [InlineData(new byte[] { 1, 2, 2, 15 })] + public void OverSubscribedCodeIsRejected(byte[] lens) + { + Assert.False(HuffmanTableBuilder.TryBuild( + new uint[InflateFormat.LitlenTableSize], lens, _symbolResults, 11, 15, new ushort[lens.Length], shrink: false, out _)); + } + + [Theory] + [InlineData(new byte[] { 2, 2, 2 })] + [InlineData(new byte[] { 2 })] + [InlineData(new byte[] { 1, 0, 2 })] + [InlineData(new byte[] { 1, 15 })] + public void IncompleteCodeIsRejected(byte[] lens) + { + Assert.False(HuffmanTableBuilder.TryBuild( + new uint[InflateFormat.LitlenTableSize], lens, _symbolResults, 11, 15, new ushort[lens.Length], shrink: false, out _)); + } + + [Fact] + public void SingleOneBitCodewordFillsTheTableWithItsSymbol() + { + uint[] table = new uint[256]; + + Assert.True(HuffmanTableBuilder.TryBuild(table, [0, 0, 1, 0], _symbolResults, 8, 15, new ushort[4], shrink: false, out _)); + + Assert.All(table, static entry => Assert.Equal((2U << 16) | (1U << 8) | 1U, entry)); + } + + [Fact] + public void EmptyCodeFillsTheTableWithSymbolZero() + { + uint[] table = new uint[256]; + Array.Fill(table, uint.MaxValue); + + Assert.True(HuffmanTableBuilder.TryBuild(table, [0, 0, 0, 0], _symbolResults, 8, 15, new ushort[4], shrink: false, out _)); + + Assert.All(table, static entry => Assert.Equal((1U << 8) | 1U, entry)); + } + + private static (int Symbol, int Length) Lookup(uint[] table, uint bits, int tableBits) + { + uint entry = table[bits & ((1U << tableBits) - 1)]; + if ((entry & InflateFormat.SubtablePointer) == 0) + { + return ((int)(entry >> 16), (int)(entry & 0xFF)); + } + int subtableBits = (int)((entry >> 8) & 0x3F); + uint subEntry = table[(entry >> 16) + ((bits >> tableBits) & ((1U << subtableBits) - 1))]; + return ((int)(subEntry >> 16), tableBits + (int)(subEntry & 0xFF)); + } + + // RFC 1951 section 3.2.2. + private static uint[] CanonicalCodewords(byte[] lens) + { + int[] counts = new int[16]; + foreach (byte len in lens) + { + counts[len]++; + } + counts[0] = 0; + uint[] next = new uint[16]; + uint code = 0; + for (int bits = 1; bits <= 15; bits++) + { + code = (code + (uint)counts[bits - 1]) << 1; + next[bits] = code; + } + uint[] codewords = new uint[lens.Length]; + for (int symbol = 0; symbol < lens.Length; symbol++) + { + if (lens[symbol] != 0) + { + codewords[symbol] = next[lens[symbol]]++; + } + } + return codewords; + } + + private static uint Reverse(uint codeword, int length) + { + uint reversed = 0; + for (int i = 0; i < length; i++) + { + reversed = (reversed << 1) | ((codeword >> i) & 1); + } + return reversed; + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Zip/InflateDifferentialTests.cs b/tests/ExcelReader.Tests/Reader/Zip/InflateDifferentialTests.cs new file mode 100644 index 00000000..337464bf --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Zip/InflateDifferentialTests.cs @@ -0,0 +1,87 @@ +using System.Buffers.Binary; +using System.IO.Compression; +using static ExcelReader.Tests.Reader.Zip.InflateTestSupport; + +namespace ExcelReader.Tests.Reader.Zip +{ + public class InflateDifferentialTests + { + public static TheoryData ShapesAndLevels() + { + var data = new TheoryData(); + foreach (DataShape shape in Enum.GetValues()) + { + foreach (CompressionLevel level in Enum.GetValues()) + { + data.Add(shape, level); + } + } + return data; + } + + [Theory] + [MemberData(nameof(ShapesAndLevels))] + public void DeflateStreamOutputRoundTrips(DataShape shape, CompressionLevel level) + { + int[] sizes = [1, 2, 7, 100, 1000, 32768, 65536, 70_000, 300_000, 1_500_000]; + foreach (int size in sizes) + { + byte[] raw = Generate(shape, size, seed: size); + byte[] stream = Deflate(raw, level); + Assert.NotEmpty(stream); + + Assert.True(raw.AsSpan().SequenceEqual(Inflate(stream, readSize: 64 * 1024)), $"size {size}"); + } + } + + [Theory] + [InlineData("sample.xlsx")] + [InlineData("RealExcel.xlsb")] + public void EveryDeflateEntryOfAProducerWrittenWorkbookMatchesDeflateStream(string fileName) + { + byte[] zip = File.ReadAllBytes(Path.Combine(AppContext.BaseDirectory, "data", fileName)); + int compared = 0; + + foreach (byte[] stream in DeflateEntries(zip)) + { + using var reference = new MemoryStream(); + using (var deflate = new DeflateStream(new MemoryStream(stream), CompressionMode.Decompress)) + { + deflate.CopyTo(reference); + } + + Assert.True(reference.ToArray().AsSpan().SequenceEqual(Inflate(stream)), $"entry {compared} of {fileName}"); + compared++; + } + + Assert.True(compared >= 5, $"{fileName} yielded only {compared} deflate entries"); + } + + private static List DeflateEntries(byte[] zip) + { + int end = zip.AsSpan().LastIndexOf("PK\x05\x06"u8); + Assert.True(end >= 0, "no end-of-central-directory record"); + int count = BinaryPrimitives.ReadUInt16LittleEndian(zip.AsSpan(end + 10)); + int position = (int)BinaryPrimitives.ReadUInt32LittleEndian(zip.AsSpan(end + 16)); + var entries = new List(); + for (int i = 0; i < count; i++) + { + ReadOnlySpan header = zip.AsSpan(position); + int method = BinaryPrimitives.ReadUInt16LittleEndian(header[10..]); + int compressedSize = (int)BinaryPrimitives.ReadUInt32LittleEndian(header[20..]); + int nameLength = BinaryPrimitives.ReadUInt16LittleEndian(header[28..]); + int extraLength = BinaryPrimitives.ReadUInt16LittleEndian(header[30..]); + int commentLength = BinaryPrimitives.ReadUInt16LittleEndian(header[32..]); + int local = (int)BinaryPrimitives.ReadUInt32LittleEndian(header[42..]); + if (method == 8) + { + ReadOnlySpan localHeader = zip.AsSpan(local); + int data = local + 30 + BinaryPrimitives.ReadUInt16LittleEndian(localHeader[26..]) + BinaryPrimitives.ReadUInt16LittleEndian(localHeader[28..]); + entries.Add(zip.AsSpan(data, compressedSize).ToArray()); + } + position += 46 + nameLength + extraLength + commentLength; + } + return entries; + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Zip/InflateStreamTests.cs b/tests/ExcelReader.Tests/Reader/Zip/InflateStreamTests.cs new file mode 100644 index 00000000..db152e6b --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Zip/InflateStreamTests.cs @@ -0,0 +1,352 @@ +using System.Diagnostics.CodeAnalysis; +using System.IO.Compression; +using ExcelReader.Core.Reader.Zip.Inflate; +using static ExcelReader.Tests.Reader.Zip.InflateTestSupport; + +namespace ExcelReader.Tests.Reader.Zip +{ + public class InflateStreamTests + { + [Fact] + public void StoredBlockIsCopiedThrough() + { + byte[] stream = [0x01, 0x05, 0x00, 0xFA, 0xFF, (byte)'h', (byte)'e', (byte)'l', (byte)'l', (byte)'o']; + + Assert.Equal("hello"u8.ToArray(), Inflate(stream)); + } + + [Fact] + public void EmptyFixedBlockProducesNothing() + { + Assert.Empty(Inflate([0x03, 0x00])); + } + + [Fact] + public void ZeroLengthStoredBlockIsSkipped() + { + byte[] stream = [0x00, 0x00, 0x00, 0xFF, 0xFF, 0x01, 0x02, 0x00, 0xFD, 0xFF, (byte)'o', (byte)'k']; + + Assert.Equal("ok"u8.ToArray(), Inflate(stream)); + } + + [Fact] + public void BytesAfterTheFinalBlockAreIgnored() + { + byte[] raw = Generate(DataShape.SheetXml, 5000, seed: 1); + byte[] stream = [.. Deflate(raw, CompressionLevel.Optimal), 0xDE, 0xAD, 0xBE, 0xEF]; + + Assert.Equal(raw, Inflate(stream)); + } + + [Fact] + public void ReservedBlockTypeIsRejected() + { + InvalidDataException ex = Assert.Throws(() => Inflate([0x07, 0x00, 0x00, 0x00])); + + Assert.Contains("reserved block type", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void StoredBlockWithMismatchedLengthFieldsIsRejected() + { + InvalidDataException ex = Assert.Throws( + () => Inflate([0x01, 0x05, 0x00, 0x00, 0x00, 1, 2, 3, 4, 5])); + + Assert.Contains("mismatched length", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void MatchReachingBeforeTheOutputStartIsRejected() + { + // Fixed block whose first symbol is a length-3 match at offset 1, with nothing before it. + InvalidDataException ex = Assert.Throws(() => Inflate([0x03, 0x02, 0x00, 0x00])); + + Assert.Contains("before the start", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void DynamicHeaderWhoseCodeLengthsOverrunIsRejected() + { + // Code-length code: symbols 0 and 18, one bit each. Two runs of 138 zeros exceed the 258 lengths declared. + byte[] stream = DynamicHeader(precodeLens: [0, 0, 1, 1], codeLengthBits: [(1, 1), (127, 7), (1, 1), (127, 7)]); + + InvalidDataException ex = Assert.Throws(() => Inflate(stream)); + + Assert.Contains("overrun", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void DynamicHeaderThatRepeatsBeforeAnyLengthIsRejected() + { + // Code-length code: symbols 0 and 16, one bit each. The first symbol is 16, "repeat the previous length". + byte[] stream = DynamicHeader(precodeLens: [1, 0, 0, 1], codeLengthBits: [(1, 1), (0, 2)]); + + InvalidDataException ex = Assert.Throws(() => Inflate(stream)); + + Assert.Contains("before defining one", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void DynamicHeaderWithOverSubscribedCodeLengthCodeIsRejected() + { + byte[] stream = DynamicHeader(precodeLens: [1, 1, 1, 1], codeLengthBits: []); + + InvalidDataException ex = Assert.Throws(() => Inflate(stream)); + + Assert.Contains("code-length code", ex.Message, StringComparison.Ordinal); + } + + [Theory] + [InlineData(int.MaxValue)] + [InlineData(1)] + public void MatchAtTheFullWindowDistanceCopiesAcrossEverySlide(int sourceStep) + { + byte[] history = Generate(DataShape.Random, 32_768, seed: 3); + byte[] expected = new byte[history.Length + (6000 * 258)]; + history.CopyTo(expected, 0); + for (int i = history.Length; i < expected.Length; i++) + { + expected[i] = expected[i - 32_768]; + } + + Assert.Equal(expected, Inflate(StoredThenFarMatches(history, matches: 6000), sourceStep: sourceStep)); + } + + [Fact] + public void MatchAtTheFullWindowDistanceWithOneByteTooFewIsRejected() + { + byte[] stream = StoredThenFarMatches(Generate(DataShape.Random, 32_767, seed: 3), matches: 1); + + InvalidDataException ex = Assert.Throws(() => Inflate(stream)); + + Assert.Contains("before the start", ex.Message, StringComparison.Ordinal); + } + + [Fact] + public void EmptyInputIsRejected() + { + Assert.Throws(() => Inflate([])); + } + + [Theory] + [InlineData(DataShape.SheetXml, CompressionLevel.Optimal)] + [InlineData(DataShape.SheetXml, CompressionLevel.Fastest)] + [InlineData(DataShape.Random, CompressionLevel.NoCompression)] + [InlineData(DataShape.Zeros, CompressionLevel.Optimal)] + [SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Seeded cut positions, not security.")] + public void StreamCutAnywhereIsRejected(DataShape shape, CompressionLevel level) + { + byte[] stream = Deflate(Generate(shape, 200_000, seed: 7), level); + var random = new Random(11); + int[] cuts = [1, 2, stream.Length / 2, stream.Length - 2, stream.Length - 1, .. Enumerable.Range(0, 40).Select(_ => random.Next(1, stream.Length))]; + + foreach (int cut in cuts) + { + Assert.Throws(() => Inflate(stream[..cut])); + } + } + + [Theory] + [InlineData(DataShape.SheetXml, CompressionLevel.Optimal)] + [InlineData(DataShape.SheetXml, CompressionLevel.Fastest)] + [InlineData(DataShape.MixedRuns, CompressionLevel.Optimal)] + public void StreamCutJustBeforeTheWindowFillsDeliversOnlyRealBytes(DataShape shape, CompressionLevel level) + { + // The window first fills at 98,304 bytes; cuts whose real output ends just short of it make + // the zero padding decode across the boundary. + byte[] raw = Generate(shape, 200_000, seed: 7); + byte[] stream = Deflate(raw, level); + int first = FirstCutReaching(stream, 98_304 - 300); + int last = FirstCutReaching(stream, 98_304); + + for (int cut = first; cut <= last; cut++) + { + byte[] delivered = DeliveredBeforeRejection(stream[..cut]); + Assert.True(raw.AsSpan().StartsWith(delivered), $"cut {cut} delivered bytes the stream never encoded"); + } + } + + [Theory] + [InlineData(1)] + [InlineData(7)] + [InlineData(4096)] + [InlineData(1 << 20)] + public void OutputIsTheSameForEveryReadSize(int readSize) + { + byte[] raw = Generate(DataShape.MixedRuns, 300_000, seed: 3); + + Assert.Equal(raw, Inflate(Deflate(raw, CompressionLevel.Optimal), readSize)); + } + + [Theory] + [InlineData(1)] + [InlineData(3)] + [InlineData(997)] + public void InputMayArriveInSmallPieces(int sourceStep) + { + byte[] raw = Generate(DataShape.SheetXml, 150_000, seed: 5); + + Assert.Equal(raw, Inflate(Deflate(raw, CompressionLevel.Optimal), readSize: 8192, sourceStep)); + } + + [Theory] + [InlineData(DataShape.SheetXml, 1)] + [InlineData(DataShape.RepeatedBlock, 997)] + [InlineData(DataShape.Random, int.MaxValue)] + public async Task ReadAsyncProducesTheSameBytesAsRead(DataShape shape, int sourceStep) + { + byte[] stream = Deflate(Generate(shape, 250_000, seed: 9), CompressionLevel.Optimal); + + byte[] sync = Inflate(stream, readSize: 5000, sourceStep); + byte[] async = await InflateAsync(stream, readSize: 5000, sourceStep, TestContext.Current.CancellationToken); + + Assert.Equal(sync, async); + } + + [Fact] + public void StreamsDecodingInParallelDoNotInterfere() + { + byte[] raw = Generate(DataShape.SheetXml, 400_000, seed: 13); + byte[] stream = Deflate(raw, CompressionLevel.Optimal); + + Parallel.For(0, 32, _ => Assert.True(raw.AsSpan().SequenceEqual(Inflate(stream, readSize: 3000)))); + } + + [Fact] + public void DisposeDisposesTheInnerStream() + { + var inner = new SteppedStream([0x03, 0x00], int.MaxValue); + + new InflateStream(inner).Dispose(); + + Assert.True(inner.Disposed); + } + + [Fact] + public void ReadAfterDisposeThrows() + { + var inflate = new InflateStream(new MemoryStream([0x03, 0x00])); + inflate.Dispose(); + + Assert.Throws(() => inflate.Read(new byte[1])); + } + + [Fact] + public void DisposeTwiceIsHarmless() + { + var inflate = new InflateStream(new MemoryStream([0x03, 0x00])); + + inflate.Dispose(); + inflate.Dispose(); + + Assert.False(inflate.CanRead); + } + + // DeflateStream's output never shrinks as the cut grows, so a binary search finds the boundary. + private static int FirstCutReaching(byte[] stream, int outputLength) + { + int low = 1; + int high = stream.Length; + while (low < high) + { + int mid = (low + high) / 2; + using var reference = new DeflateStream(new MemoryStream(stream, 0, mid), CompressionMode.Decompress); + using var output = new MemoryStream(); + reference.CopyTo(output); + if (output.Length >= outputLength) + { + high = mid; + } + else + { + low = mid + 1; + } + } + return low; + } + + private static byte[] DeliveredBeforeRejection(byte[] stream) + { + using var inflate = new InflateStream(new MemoryStream(stream)); + using var output = new MemoryStream(); + byte[] buffer = new byte[4096]; + try + { + int read; + while ((read = inflate.Read(buffer)) > 0) + { + output.Write(buffer, 0, read); + } + } + catch (InvalidDataException) + { + return output.ToArray(); + } + throw new InvalidOperationException("A truncated stream was accepted."); + } + + // A stored block holding the history, then a final fixed block of length-258 matches at offset + // 32,768: symbol 285 and offset code 29 with all 13 extra bits set. Huffman codes go in bit-reversed. + private static byte[] StoredThenFarMatches(byte[] history, int matches) + { + var bits = new BitWriter(); + bits.Write(1, 1); + bits.Write(1, 2); + for (int i = 0; i < matches; i++) + { + bits.Write(0b1010_0011, 8); + bits.Write(0b1_0111, 5); + bits.Write(8191, 13); + } + bits.Write(0, 7); + int length = history.Length; + return [0x00, (byte)length, (byte)(length >> 8), (byte)~length, (byte)(~length >> 8), .. history, .. bits.ToArray()]; + } + + // A final dynamic block declaring 257 literal/length codes and 1 offset code, with the given + // lengths for the first four code-length-code symbols (16, 17, 18, 0) and raw bits after them. + private static byte[] DynamicHeader(int[] precodeLens, (int Value, int Count)[] codeLengthBits) + { + var bits = new BitWriter(); + bits.Write(1, 1); + bits.Write(2, 2); + bits.Write(0, 5); + bits.Write(0, 5); + bits.Write(precodeLens.Length - 4, 4); + foreach (int len in precodeLens) + { + bits.Write(len, 3); + } + foreach ((int value, int count) in codeLengthBits) + { + bits.Write(value, count); + } + return bits.ToArray(); + } + + private sealed class BitWriter + { + private readonly List _bytes = []; + private int _used = 8; + + internal void Write(int value, int count) + { + for (int i = 0; i < count; i++) + { + if (_used == 8) + { + _bytes.Add(0); + _used = 0; + } + _bytes[^1] |= (byte)(((value >> i) & 1) << _used); + _used++; + } + } + + internal byte[] ToArray() + { + return [.. _bytes]; + } + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Zip/InflateTestSupport.cs b/tests/ExcelReader.Tests/Reader/Zip/InflateTestSupport.cs new file mode 100644 index 00000000..fd12e36a --- /dev/null +++ b/tests/ExcelReader.Tests/Reader/Zip/InflateTestSupport.cs @@ -0,0 +1,202 @@ +using System.Diagnostics.CodeAnalysis; +using System.IO.Compression; +using System.Text; +using ExcelReader.Core.Reader.Zip.Inflate; + +namespace ExcelReader.Tests.Reader.Zip +{ + [SuppressMessage("Security", "CA5394:Do not use insecure randomness", Justification = "Seeded test data, not security.")] + internal static class InflateTestSupport + { + internal static byte[] Deflate(byte[] raw, CompressionLevel level) + { + using var compressed = new MemoryStream(); + using (var deflate = new DeflateStream(compressed, level, leaveOpen: true)) + { + deflate.Write(raw); + } + return compressed.ToArray(); + } + + internal static byte[] Inflate(byte[] compressed, int readSize = 4096, int sourceStep = int.MaxValue) + { + using var inflate = new InflateStream(new SteppedStream(compressed, sourceStep)); + using var output = new MemoryStream(); + byte[] buffer = new byte[readSize]; + int read; + while ((read = inflate.Read(buffer)) > 0) + { + output.Write(buffer, 0, read); + } + return output.ToArray(); + } + + internal static async Task InflateAsync(byte[] compressed, int readSize, int sourceStep, CancellationToken ct) + { + using var inflate = new InflateStream(new SteppedStream(compressed, sourceStep)); + using var output = new MemoryStream(); + byte[] buffer = new byte[readSize]; + int read; + while ((read = await inflate.ReadAsync(buffer, ct)) > 0) + { + output.Write(buffer, 0, read); + } + return output.ToArray(); + } + + internal static byte[] Generate(DataShape shape, int size, int seed) + { + var random = new Random(seed); + byte[] data = new byte[size]; + switch (shape) + { + case DataShape.Zeros: + break; + case DataShape.Random: + random.NextBytes(data); + break; + case DataShape.SheetXml: + FillSheetXml(data, random); + break; + case DataShape.ShortPeriods: + FillShortPeriods(data, random); + break; + case DataShape.RepeatedBlock: + FillRepeatedBlock(data, random); + break; + default: + FillMixedRuns(data, random); + break; + } + return data; + } + + private static void FillSheetXml(byte[] data, Random random) + { + var xml = new StringBuilder(); + for (int row = 1; xml.Length < data.Length; row++) + { + xml.Append(System.Globalization.CultureInfo.InvariantCulture, + $"{random.NextDouble() * 1000:F4}{random.Next(500)}"); + } + Encoding.ASCII.GetBytes(xml.ToString(0, data.Length), data); + } + + // Runs whose period is 1 to 9 bytes, so matches land on every short offset. + private static void FillShortPeriods(byte[] data, Random random) + { + int position = 0; + while (position < data.Length) + { + int period = random.Next(1, 10); + int run = Math.Min(data.Length - position, random.Next(period, 600)); + for (int i = 0; i < run; i++) + { + data[position + i] = i < period ? (byte)random.Next(256) : data[position + i - period]; + } + position += run; + } + } + + // One incompressible 32,768-byte block repeated, so matches sit at the far edge of the window. + private static void FillRepeatedBlock(byte[] data, Random random) + { + const int Block = 32768; + random.NextBytes(data.AsSpan(0, Math.Min(Block, data.Length))); + for (int i = Block; i < data.Length; i++) + { + data[i] = data[i - Block]; + } + } + + private static void FillMixedRuns(byte[] data, Random random) + { + int position = 0; + while (position < data.Length) + { + int run = Math.Min(data.Length - position, random.Next(1, 400)); + if (random.Next(3) == 0) + { + random.NextBytes(data.AsSpan(position, run)); + } + else + { + data.AsSpan(position, run).Fill((byte)random.Next(256)); + } + position += run; + } + } + } + + public enum DataShape + { + Zeros, + Random, + SheetXml, + ShortPeriods, + RepeatedBlock, + MixedRuns, + } + + internal sealed class SteppedStream(byte[] data, int step) : Stream + { + private int _position; + + internal bool Disposed { get; private set; } + + public override bool CanRead => true; + public override bool CanSeek => false; + public override bool CanWrite => false; + public override long Length => data.Length; + + public override long Position + { + get => _position; + set => throw new NotSupportedException(); + } + + public override int Read(byte[] buffer, int offset, int count) + { + return Read(buffer.AsSpan(offset, count)); + } + + public override int Read(Span buffer) + { + int count = Math.Min(Math.Min(step, buffer.Length), data.Length - _position); + data.AsSpan(_position, count).CopyTo(buffer); + _position += count; + return count; + } + + public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + await Task.Yield(); + return Read(buffer.Span); + } + + public override void Flush() + { + } + + public override long Seek(long offset, SeekOrigin origin) + { + throw new NotSupportedException(); + } + + public override void SetLength(long value) + { + throw new NotSupportedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + throw new NotSupportedException(); + } + + protected override void Dispose(bool disposing) + { + Disposed = true; + base.Dispose(disposing); + } + } +} diff --git a/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs index 8a765067..4328dfc1 100644 --- a/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs +++ b/tests/ExcelReader.Tests/Reader/Zip/ZipIndexTests.cs @@ -859,6 +859,89 @@ public void ZipArchiveEntryOpenSilentlyTruncatesAtDeclaredLengthUnderOverDeliver Assert.Equal(-1, stream.ReadByte()); } + [Fact] + public void DeflatedEntryCutShortIsRejectedWhenOpenedAsAPart() + { + byte[] zipBytes = BuildZipWithCutDeflatedEntry("data.xml"); + + using ZipIndex index = ZipIndex.Create(zipBytes, ExcelReaderOptions.Default); + Assert.True(index.TryGetEntry("data.xml"u8, out ZipEntryRef entry)); + Assert.Equal((ushort)8, entry.Method); + Assert.Throws(() => index.OpenPart(entry, new DecompressedByteCounter(0))); + } + + [Fact] + public void DeflatedEntryCutShortIsRejectedWhenStreamedInsteadOfEndingEarly() + { + byte[] zipBytes = BuildZipWithCutDeflatedEntry("data.xml"); + + using ZipIndex index = ZipIndex.Create(zipBytes, ExcelReaderOptions.Default); + Assert.True(index.TryGetEntry("data.xml"u8, out ZipEntryRef entry)); + using LimitedReadStream stream = index.OpenEntryStream(entry, new DecompressedByteCounter(0), ExcelReaderOptions.Default); + Assert.Throws(() => stream.CopyTo(Stream.Null)); + } + + [Theory] + [InlineData(1)] + [InlineData(-1)] + public void StreamedEntryWhoseLengthDiffersFromItsDeclaredSizeIsRejected(int declaredDelta) + { + byte[] payload = Encoding.UTF8.GetBytes(string.Concat(Enumerable.Repeat("12345", 4000))); + byte[] zipBytes = BuildZipWithOneEntry("data.xml", payload, CompressionLevel.Optimal); + PatchCentralDirectoryUInt32(zipBytes, "data.xml", fieldOffset: 24, (uint)(payload.Length + declaredDelta)); + + using ZipIndex index = ZipIndex.Create(zipBytes, ExcelReaderOptions.Default); + Assert.True(index.TryGetEntry("data.xml"u8, out ZipEntryRef entry)); + using LimitedReadStream stream = index.OpenEntryStream(entry, new DecompressedByteCounter(0), ExcelReaderOptions.Default); + + InvalidDataException ex = Assert.Throws(() => stream.CopyTo(Stream.Null)); + Assert.Contains(declaredDelta > 0 ? "less data" : "more data", ex.Message, StringComparison.Ordinal); + } + + [Theory] + [InlineData(1)] + [InlineData(-1)] + public async Task StreamedEntryWhoseLengthDiffersFromItsDeclaredSizeIsRejectedAsync(int declaredDelta) + { + byte[] payload = Encoding.UTF8.GetBytes(string.Concat(Enumerable.Repeat("12345", 4000))); + byte[] zipBytes = BuildZipWithOneEntry("data.xml", payload, CompressionLevel.Optimal); + PatchCentralDirectoryUInt32(zipBytes, "data.xml", fieldOffset: 24, (uint)(payload.Length + declaredDelta)); + CancellationToken ct = TestContext.Current.CancellationToken; + + using ZipIndex index = ZipIndex.Create(zipBytes, ExcelReaderOptions.Default); + Assert.True(index.TryGetEntry("data.xml"u8, out ZipEntryRef entry)); + await using LimitedReadStream stream = await index.OpenEntryStreamAsync(entry, new DecompressedByteCounter(0), ExcelReaderOptions.Default, ct); + + await Assert.ThrowsAsync(() => stream.CopyToAsync(Stream.Null, ct)); + } + + [Fact] + public void StreamedEntryOfExactlyItsDeclaredSizeReadsToTheEnd() + { + byte[] payload = Encoding.UTF8.GetBytes(string.Concat(Enumerable.Repeat("12345", 4000))); + byte[] zipBytes = BuildZipWithOneEntry("data.xml", payload, CompressionLevel.Optimal); + + using ZipIndex index = ZipIndex.Create(zipBytes, ExcelReaderOptions.Default); + Assert.True(index.TryGetEntry("data.xml"u8, out ZipEntryRef entry)); + using LimitedReadStream stream = index.OpenEntryStream(entry, new DecompressedByteCounter(0), ExcelReaderOptions.Default); + using var copy = new MemoryStream(); + stream.CopyTo(copy); + + Assert.Equal(payload, copy.ToArray()); + Assert.Equal(0, stream.Read(new byte[16])); + } + + private static byte[] BuildZipWithCutDeflatedEntry(string entryName) + { + byte[] payload = Encoding.UTF8.GetBytes(string.Concat(Enumerable.Repeat("12345", 4000))); + byte[] zipBytes = BuildZipWithOneEntry(entryName, payload, CompressionLevel.Optimal); + int cdOffset = FindCentralDirectoryOffset(zipBytes, entryName); + uint compressedSize = BinaryPrimitives.ReadUInt32LittleEndian(zipBytes.AsSpan(cdOffset + 20, 4)); + Assert.True(compressedSize > 8); + PatchCentralDirectoryUInt32(zipBytes, entryName, fieldOffset: 20, compressedSize / 2); + return zipBytes; + } + private static uint ReadDeclaredUncompressedSize(byte[] zipBytes, string entryName) { int cdOffset = FindCentralDirectoryOffset(zipBytes, entryName); diff --git a/tests/ExcelReader.Tests/TestUtils.cs b/tests/ExcelReader.Tests/TestUtils.cs index 7e6653b8..ec52bb1c 100644 --- a/tests/ExcelReader.Tests/TestUtils.cs +++ b/tests/ExcelReader.Tests/TestUtils.cs @@ -257,7 +257,6 @@ internal static Action FailOnce(Exception failure) }; } - // Every span/async write on a derived MemoryStream funnels into this overload. public override void Write(byte[] buffer, int offset, int count) { OnWrite?.Invoke(); @@ -447,7 +446,8 @@ internal static MemoryStream BuildMultiSheet( (string Name, string Rows)[] sheets, string? sharedStrings = null, string? styles = null, - bool date1904 = false) + bool date1904 = false, + (string Path, string Content)[]? extraParts = null) { var sheetXml = new string[sheets.Length]; var relXml = new string[sheets.Length]; @@ -478,6 +478,10 @@ internal static MemoryStream BuildMultiSheet( $"""""", StringComparison.Ordinal); Write(zip, "xl/styles.xml", $"""{withNs}"""); } + foreach ((string path, string content) in extraParts ?? []) + { + Write(zip, path, content); + } } ms.Position = 0; return ms; diff --git a/tests/ExcelReader.Tests/Writer/Csv/CsvWriterOptionsTests.cs b/tests/ExcelReader.Tests/Writer/Csv/CsvWriterOptionsTests.cs index e5c17422..091eb792 100644 --- a/tests/ExcelReader.Tests/Writer/Csv/CsvWriterOptionsTests.cs +++ b/tests/ExcelReader.Tests/Writer/Csv/CsvWriterOptionsTests.cs @@ -80,7 +80,7 @@ public void Should_RoundTripThroughTheReader_When_BothAreGivenTheSameDialect() }); var options = CsvReaderOptions.Default with { Delimiter = (byte)';', Encoding = Encoding.Latin1 }; - using CsvReader reader = Excel.FromCsv(written.AsMemory(), options); + using CsvReader reader = Excel.FromCsv(written.AsMemory(), new ExcelReaderOptions { Csv = options }); using CsvReader.Enumerator rows = reader.FirstSheet.GetEnumerator(); Assert.True(rows.MoveNext()); diff --git a/tests/ExcelReader.Tests/Writer/Csv/CsvWriterTests.cs b/tests/ExcelReader.Tests/Writer/Csv/CsvWriterTests.cs index 63e027f1..7973709c 100644 --- a/tests/ExcelReader.Tests/Writer/Csv/CsvWriterTests.cs +++ b/tests/ExcelReader.Tests/Writer/Csv/CsvWriterTests.cs @@ -246,7 +246,7 @@ public void NumericFieldContainingQuoteByteIsEscaped() } ms.Position = 0; - using IExcelWorkbook reader = Excel.FromCsv(ms, options: new CsvReaderOptions { Quote = (byte)'5' }); + using IExcelWorkbook reader = Excel.FromCsv(ms, options: new ExcelReaderOptions { Csv = new CsvReaderOptions { Quote = (byte)'5' } }); using IExcelRowEnumerator e = reader.FirstSheet.GetEnumerator(); Assert.True(e.MoveNext()); Assert.Equal("155", e.Current[0].GetString()); @@ -340,7 +340,7 @@ public void WrittenCsvRoundTripsThroughCsvReader() Assert.Equal("has, comma", row2[0].GetString()); Assert.Equal(CellType.ExcelString, row2[1].Type); Assert.Equal("true", row2[1].GetString()); - Assert.True(row2[2].TryGetDateTime(out DateTime readBack)); + Assert.True(row2[2].TryGetDateTime(isDate1904: false, out DateTime readBack)); Assert.Equal(date, readBack); Assert.Equal(date.ToString("O"), row2[2].GetString()); } diff --git a/tests/ExcelReader.Tests/Writer/Tables/TableBooks.cs b/tests/ExcelReader.Tests/Writer/Tables/TableBooks.cs new file mode 100644 index 00000000..10ece1ff --- /dev/null +++ b/tests/ExcelReader.Tests/Writer/Tables/TableBooks.cs @@ -0,0 +1,164 @@ +using System.IO.Compression; +using ExcelReader.Core.Writer; +using ExcelReader.Core.Writer.Xlsb; +using ExcelReader.Core.Writer.Xlsx; + +namespace ExcelReader.Tests.Writer.Tables +{ + internal interface ITableBook + { + void AddSheet(string name); + + void BeginTable(string name, string[] columns, ExcelTableOptions? options = null); + + ValueTask BeginTableAsync(string name, string[] columns, ExcelTableOptions? options = null); + + void Row(params object?[] values); + + void EndTable(); + + void EndSheet(); + + byte[] Finish(); + + ValueTask FinishAsync(); + } + + internal sealed class TableBook(IWorkbookWriter workbook, MemoryStream stream) : ITableBook + where TSheet : class, ITableSheetWriter + where TRow : IRowWriter + { + private TSheet? _sheet; + + private TSheet Sheet + { + get + { + return _sheet ?? throw new InvalidOperationException("No sheet added."); + } + } + + public void AddSheet(string name) + { + _sheet = workbook.AddSheet(name); + } + + public void BeginTable(string name, string[] columns, ExcelTableOptions? options = null) + { + Sheet.BeginTable(name, columns, options); + } + + public ValueTask BeginTableAsync(string name, string[] columns, ExcelTableOptions? options = null) + { + return Sheet.BeginTableAsync(name, columns, options, TestContext.Current.CancellationToken); + } + + public void Row(params object?[] values) + { + using TRow row = Sheet.StartRow(); + foreach (object? value in values) + { + WriteValue(row, value); + } + } + + public void EndTable() + { + Sheet.EndTable(); + } + + public void EndSheet() + { + Sheet.End(); + _sheet = null; + } + + public byte[] Finish() + { + workbook.End(); + workbook.Dispose(); + return stream.ToArray(); + } + + public async ValueTask FinishAsync() + { + CancellationToken ct = TestContext.Current.CancellationToken; + if (_sheet is not null) + { + await _sheet.EndAsync(ct); + _sheet = null; + } + await workbook.EndAsync(ct); + await workbook.DisposeAsync(); + return stream.ToArray(); + } + + private static void WriteValue(TRow row, object? value) + { + switch (value) + { + case null: + row.Skip(); + break; + case string text: + row.Write(text); + break; + case int number: + row.Write(number); + break; + case double real: + row.Write(real); + break; + default: + throw new ArgumentException($"Unsupported cell value {value.GetType()}.", nameof(value)); + } + } + } + + public static class TableBooks + { + public static TheoryData Formats { get; } = new() { "xlsx", "xlsb" }; + + internal static ITableBook Create(string format) + { + MemoryStream stream = new(); + return format switch + { + "xlsx" => new TableBook(XlsxWorkbookWriter.Create(stream, leaveOpen: true), stream), + "xlsb" => new TableBook(XlsbWorkbookWriter.Create(stream, leaveOpen: true), stream), + _ => throw new ArgumentOutOfRangeException(nameof(format)), + }; + } + + internal static string Extension(string format) + { + return string.Equals(format, "xlsx", StringComparison.Ordinal) ? "xml" : "bin"; + } + + internal static byte[] Entry(byte[] package, string name) + { + using ZipArchive zip = new(new MemoryStream(package), ZipArchiveMode.Read); + ZipArchiveEntry entry = zip.GetEntry(name) ?? throw new InvalidOperationException($"Package has no entry '{name}'."); + using Stream content = entry.Open(); + using MemoryStream copy = new(); + content.CopyTo(copy); + return copy.ToArray(); + } + + internal static IReadOnlyList EntryNames(byte[] package) + { + using ZipArchive zip = new(new MemoryStream(package), ZipArchiveMode.Read); + return [.. zip.Entries.Select(static e => e.FullName)]; + } + + internal static Dictionary Entries(byte[] package) + { + Dictionary entries = new(StringComparer.Ordinal); + foreach (string name in EntryNames(package)) + { + entries[name] = Entry(package, name); + } + return entries; + } + } +} diff --git a/tests/ExcelReader.Tests/Writer/Tables/TableRecordWriterTests.cs b/tests/ExcelReader.Tests/Writer/Tables/TableRecordWriterTests.cs new file mode 100644 index 00000000..41e7f771 --- /dev/null +++ b/tests/ExcelReader.Tests/Writer/Tables/TableRecordWriterTests.cs @@ -0,0 +1,107 @@ +using ExcelReader.Core.Parser; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Writer; +using ExcelReader.Core.Writer.Xlsb; +using ExcelReader.Core.Writer.Xlsx; + +namespace ExcelReader.Tests.Writer.Tables +{ + public class TableRecordWriterTests + { + public sealed class Sale + { + public string? Product { get; set; } + + public int Qty { get; set; } + + public double Price { get; set; } + } + + private static readonly Sale[] Sales = + [ + new() { Product = "Pen", Qty = 10, Price = 2.5 }, + new() { Product = "Notebook", Qty = 4, Price = 18.9 }, + ]; + + [Fact] + public async Task XlsxRecordsRoundTripAsATable() + { + using MemoryStream stream = new(); + await using (XlsxWorkbookWriter workbook = XlsxWorkbookWriter.Create(stream, leaveOpen: true)) + { + XlsxSheetWriter sheet = workbook.AddSheet("Report"); + await sheet.WriteTableAsync(Sales, "Sales", ExcelRecordLayout.FromAttributes(), + ExcelTableOptions.Default with { FirstColumn = 1 }, TestContext.Current.CancellationToken); + } + + AssertSalesTable(stream.ToArray(), "B1:D3"); + } + + [Fact] + public async Task XlsbAsyncRecordsRoundTripAsATable() + { + using MemoryStream stream = new(); + await using (XlsbWorkbookWriter workbook = XlsbWorkbookWriter.Create(stream, leaveOpen: true)) + { + XlsbSheetWriter sheet = workbook.AddSheet("Report"); + await sheet.WriteTableAsync(Stream(), "Sales", ExcelRecordLayout.FromAttributes(), + ct: TestContext.Current.CancellationToken); + } + + AssertSalesTable(stream.ToArray(), "A1:C3"); + } + + [Fact] + public async Task TableIsClosedSoAnotherCanFollow() + { + using MemoryStream stream = new(); + await using (XlsxWorkbookWriter workbook = XlsxWorkbookWriter.Create(stream, leaveOpen: true)) + { + XlsxSheetWriter sheet = workbook.AddSheet("Report"); + await sheet.WriteTableAsync(Sales, "First", ExcelRecordLayout.FromAttributes(), ct: TestContext.Current.CancellationToken); + await sheet.WriteTableAsync(Sales, "Second", ExcelRecordLayout.FromAttributes(), ct: TestContext.Current.CancellationToken); + } + + using IExcelWorkbook read = Excel.Open(new ReadOnlyMemory(stream.ToArray())); + Assert.Equal([("First", "A1:C3"), ("Second", "A4:C6")], read.Tables.Select(static t => (t.Name, t.Ref))); + } + + [Fact] + public async Task EmptyTableDoesNotOverlapTheNextOne() + { + using MemoryStream stream = new(); + await using (XlsxWorkbookWriter workbook = XlsxWorkbookWriter.Create(stream, leaveOpen: true)) + { + XlsxSheetWriter sheet = workbook.AddSheet("Report"); + await sheet.WriteTableAsync(Array.Empty(), "Empty", ExcelRecordLayout.FromAttributes(), ct: TestContext.Current.CancellationToken); + await sheet.WriteTableAsync(Sales, "Full", ExcelRecordLayout.FromAttributes(), ct: TestContext.Current.CancellationToken); + } + + using IExcelWorkbook read = Excel.Open(new ReadOnlyMemory(stream.ToArray())); + Assert.Equal([("Empty", "A1:C2"), ("Full", "A3:C5")], read.Tables.Select(static t => (t.Name, t.Ref))); + } + + private static async IAsyncEnumerable Stream() + { + foreach (Sale sale in Sales) + { + await Task.Yield(); + yield return sale; + } + } + + private static void AssertSalesTable(byte[] package, string reference) + { + using IExcelWorkbook read = Excel.Open(new ReadOnlyMemory(package)); + ExcelTable table = Assert.Single(read.Tables); + Assert.Equal(("Sales", reference), (table.Name, table.Ref)); + Assert.Equal(["Product", "Qty", "Price"], table.ColumnNames); + List<(string?, int, double)> rows = []; + foreach (Sale sale in ExcelParser.FromAttributes().Parse(table.AsSheet())) + { + rows.Add((sale.Product, sale.Qty, sale.Price)); + } + Assert.Equal([("Pen", 10, 2.5), ("Notebook", 4, 18.9)], rows); + } + } +} diff --git a/tests/ExcelReader.Tests/Writer/Tables/TableValidationTests.cs b/tests/ExcelReader.Tests/Writer/Tables/TableValidationTests.cs new file mode 100644 index 00000000..8f48298f --- /dev/null +++ b/tests/ExcelReader.Tests/Writer/Tables/TableValidationTests.cs @@ -0,0 +1,172 @@ +using ExcelReader.Core.Reader; +using ExcelReader.Core.Writer; +using ExcelReader.Core.Writer.Internal; + +namespace ExcelReader.Tests.Writer.Tables +{ + public class TableValidationTests + { + [Theory] + [InlineData("Sales")] + [InlineData("_private")] + [InlineData("\\back")] + [InlineData("Q1.Results_2026")] + [InlineData("Vendas")] + [InlineData("ABCD1")] + [InlineData("Ação")] + public void ValidNamesPass(string name) + { + string[] columns = TableValidation.Validate(name, ["A"], ExcelTableOptions.Default); + + Assert.Equal(["A"], columns); + } + + [Theory] + [InlineData("")] + [InlineData("1abc")] + [InlineData(".dot")] + [InlineData("has space")] + [InlineData("dash-name")] + [InlineData("A1")] + [InlineData("abc1")] + [InlineData("XFD1048576")] + [InlineData("R1C1")] + [InlineData("r")] + [InlineData("C")] + [InlineData("RC")] + [InlineData("R12")] + [InlineData("c3")] + public void InvalidNamesThrow(string name) + { + Assert.Throws(() => TableValidation.Validate(name, ["A"], ExcelTableOptions.Default)); + } + + [Fact] + public void NullAndOverlongNamesThrow() + { + Assert.Throws(() => TableValidation.Validate(null!, ["A"], ExcelTableOptions.Default)); + Assert.Throws(() => TableValidation.Validate(new string('a', 256), ["A"], ExcelTableOptions.Default)); + Assert.Equal(["A"], TableValidation.Validate(new string('a', 255), ["A"], ExcelTableOptions.Default)); + } + + [Fact] + public void InvalidColumnsThrow() + { + Assert.Throws(() => TableValidation.Validate("T", null!, ExcelTableOptions.Default)); + Assert.Throws(() => TableValidation.Validate("T", [], ExcelTableOptions.Default)); + Assert.Throws(() => TableValidation.Validate("T", ["a", ""], ExcelTableOptions.Default)); + Assert.Throws(() => TableValidation.Validate("T", ["a", null!], ExcelTableOptions.Default)); + Assert.Throws(() => TableValidation.Validate("T", ["Qty", "qty"], ExcelTableOptions.Default)); + Assert.Throws(() => TableValidation.Validate("T", ["a\nb"], ExcelTableOptions.Default)); + Assert.Throws(() => TableValidation.Validate("T", [new string('c', 256)], ExcelTableOptions.Default)); + } + + [Fact] + public void ColumnsMustFitTheSheet() + { + string[] two = ["a", "b"]; + + Assert.Equal(two, TableValidation.Validate("T", two, ExcelTableOptions.Default with { FirstColumn = 16382 })); + Assert.Throws(() => TableValidation.Validate("T", two, ExcelTableOptions.Default with { FirstColumn = 16383 })); + Assert.Throws(() => TableValidation.Validate("T", two, ExcelTableOptions.Default with { FirstColumn = -1 })); + } + + [Theory] + [InlineData("TableStyleLight1")] + [InlineData("TableStyleLight21")] + [InlineData("TableStyleMedium28")] + [InlineData("TableStyleDark11")] + [InlineData(null)] + public void BuiltInStylesPass(string? style) + { + Assert.Equal(["A"], TableValidation.Validate("T", ["A"], ExcelTableOptions.Default with { StyleName = style })); + } + + [Theory] + [InlineData("TableStyleLight0")] + [InlineData("TableStyleLight22")] + [InlineData("TableStyleMedium29")] + [InlineData("TableStyleDark12")] + [InlineData("TableStyleLight01")] + [InlineData("tablestylelight1")] + [InlineData("MyStyle")] + [InlineData("")] + public void UnknownStylesThrow(string style) + { + Assert.Throws(() => TableValidation.Validate("T", ["A"], ExcelTableOptions.Default with { StyleName = style })); + } + + [Fact] + public void ValidateReturnsACopyOfTheColumns() + { + List source = ["a", "b"]; + string[] copy = TableValidation.Validate("T", source, ExcelTableOptions.Default); + source[0] = "changed"; + + Assert.Equal(["a", "b"], copy); + } + + [Fact] + public void RegistryClaimsNamesCaseInsensitivelyAndNumbersTables() + { + TableRegistry registry = new(); + + Assert.Equal(1, registry.Claim("Sales")); + Assert.Equal(2, registry.Claim("Stock")); + Assert.Throws(() => registry.RequireAvailable("sales")); + Assert.Throws(() => registry.Claim("STOCK")); + Assert.Equal(3, registry.Claim("Other")); + } + + [Fact] + public void TrackerGivesHeaderOnlyTablesOneDataRow() + { + TableTracker tracker = new(); + tracker.Begin(1, "T", ["A"], ExcelTableOptions.Default, headerRow: 4); + tracker.End(lastWrittenRow: 4); + + WrittenTable table = Assert.Single(tracker.Finished); + Assert.Equal((4, 5, "A5:A6"), (table.HeaderRow, table.LastRow, table.Ref)); + } + + [Fact] + public void TrackerRefusesTwoOpenTablesAndEndWithoutOpen() + { + TableTracker tracker = new(); + Assert.Throws(() => tracker.End(0)); + + tracker.Begin(1, "T", ["A", "B"], ExcelTableOptions.Default with { FirstColumn = 2 }, headerRow: 0); + Assert.Throws(() => tracker.RequireNoneOpen()); + tracker.CloseOpen(lastWrittenRow: 3); + tracker.CloseOpen(lastWrittenRow: 9); + + WrittenTable table = Assert.Single(tracker.Finished); + Assert.Equal((2, 3, "C1:D4"), (table.FirstColumn, table.LastColumn, table.Ref)); + } + + [Fact] + public void TableNeedsRoomForItsHeaderAndOneDataRow() + { + TableTracker.RequireRoom(1_048_574); + + Assert.Throws(() => TableTracker.RequireRoom(1_048_575)); + } + + [Fact] + public void SheetRelsListOneRelationshipPerTable() + { + WrittenTable first = new(3, "A", ["x"], ExcelTableOptions.Default, 0, 1); + WrittenTable second = new(4, "B", ["y"], ExcelTableOptions.Default, 3, 4); + + string xml = TablePackage.SheetRelsXml([first, second], "bin"); + + Assert.Equal( + "" + + "" + + "" + + "" + + "", + xml); + } + } +} diff --git a/tests/ExcelReader.Tests/Writer/Tables/TableWriterTests.cs b/tests/ExcelReader.Tests/Writer/Tables/TableWriterTests.cs new file mode 100644 index 00000000..3075defb --- /dev/null +++ b/tests/ExcelReader.Tests/Writer/Tables/TableWriterTests.cs @@ -0,0 +1,262 @@ +using ExcelReader.Core.Parser; +using ExcelReader.Core.Reader; +using ExcelReader.Core.Reader.Internal; +using ExcelReader.Core.Writer; + +namespace ExcelReader.Tests.Writer.Tables +{ + public class TableWriterTests + { + private sealed class StockRow + { + public string? Product { get; set; } + + public int Qty { get; set; } + } + + [Theory] + [MemberData(nameof(TableBooks.Formats), MemberType = typeof(TableBooks))] + public void SingleTableRoundTrips(string format) + { + ITableBook book = TableBooks.Create(format); + book.AddSheet("Report"); + book.BeginTable("Stock", ["Product", "Qty"]); + book.Row("Pen", 100); + book.Row("Notebook", 40); + book.EndTable(); + + using IExcelWorkbook workbook = Read(book.Finish()); + + ExcelTable table = Assert.Single(workbook.Tables); + Assert.Equal(("Stock", "A1:B3", "TableStyleMedium2"), (table.Name, table.Ref, table.StyleName)); + Assert.Equal((1, 0, "Report"), (table.HeaderRowCount, table.TotalsRowCount, table.Sheet.Name)); + Assert.Equal(["Product", "Qty"], table.ColumnNames); + Assert.Equal([("Pen", 100), ("Notebook", 40)], ParseStock(table)); + } + + [Theory] + [MemberData(nameof(TableBooks.Formats), MemberType = typeof(TableBooks))] + public void TableAtAnOffsetColumnRoundTrips(string format) + { + ITableBook book = TableBooks.Create(format); + book.AddSheet("Report"); + book.BeginTable("Stock", ["Product", "Qty"], ExcelTableOptions.Default with { FirstColumn = 2 }); + book.Row(null, null, "Pen", 100); + book.Row(null, null, "Notebook", 40); + + using IExcelWorkbook workbook = Read(book.Finish()); + + ExcelTable table = Assert.Single(workbook.Tables); + Assert.Equal("C1:D3", table.Ref); + Assert.Equal([("Pen", 100), ("Notebook", 40)], ParseStock(table)); + } + + [Theory] + [MemberData(nameof(TableBooks.Formats), MemberType = typeof(TableBooks))] + public void TablesInSequenceAndAcrossSheetsGetWorkbookWideIds(string format) + { + ITableBook book = TableBooks.Create(format); + book.AddSheet("One"); + book.BeginTable("First", ["A"]); + book.Row("x"); + book.EndTable(); + book.Row("between"); + book.BeginTable("Second", ["B"]); + book.Row("y"); + book.EndTable(); + book.EndSheet(); + book.AddSheet("Two"); + book.BeginTable("Third", ["C"]); + book.Row("z"); + byte[] package = book.Finish(); + + using IExcelWorkbook workbook = Read(package); + + Assert.Equal( + [("First", 0, "A1:A2"), ("Second", 0, "A4:A5"), ("Third", 1, "A1:A2")], + workbook.Tables.Select(static t => (t.Name, t.Sheet.Index, t.Ref))); + string ext = TableBooks.Extension(format); + IReadOnlyList names = TableBooks.EntryNames(package); + Assert.Contains($"xl/tables/table1.{ext}", names, StringComparer.Ordinal); + Assert.Contains($"xl/tables/table2.{ext}", names, StringComparer.Ordinal); + Assert.Contains($"xl/tables/table3.{ext}", names, StringComparer.Ordinal); + } + + [Theory] + [MemberData(nameof(TableBooks.Formats), MemberType = typeof(TableBooks))] + public void HeaderOnlyTableGetsOneEmptyDataRow(string format) + { + ITableBook book = TableBooks.Create(format); + book.AddSheet("Report"); + book.BeginTable("Empty", ["Only"]); + book.EndTable(); + + using IExcelWorkbook workbook = Read(book.Finish()); + + Assert.Equal("A1:A2", Assert.Single(workbook.Tables).Ref); + } + + [Theory] + [MemberData(nameof(TableBooks.Formats), MemberType = typeof(TableBooks))] + public void TableAfterAHeaderOnlyTableStartsBelowItsReservedRow(string format) + { + ITableBook book = TableBooks.Create(format); + book.AddSheet("Report"); + book.BeginTable("A", ["H"]); + book.EndTable(); + book.BeginTable("B", ["H"]); + book.Row("x"); + + using IExcelWorkbook workbook = Read(book.Finish()); + + Assert.Equal( + [("A", "A1:A2"), ("B", "A3:A4")], + workbook.Tables.Select(static t => (t.Name, t.Ref))); + } + + [Theory] + [MemberData(nameof(TableBooks.Formats), MemberType = typeof(TableBooks))] + public void RowAfterAHeaderOnlyTableLandsOutsideIt(string format) + { + ITableBook book = TableBooks.Create(format); + book.AddSheet("Report"); + book.BeginTable("A", ["H"]); + book.EndTable(); + book.Row("x"); + + using IExcelWorkbook workbook = Read(book.Finish()); + + Assert.Equal("A1:A2", Assert.Single(workbook.Tables).Ref); + using IExcelRowEnumerator rows = workbook.SheetAt(0).GetEnumerator(); + IRowIndexedEnumerator indexed = (IRowIndexedEnumerator)rows; + indexed.EnableRowIndex(); + int lastIndex = -1; + string? lastText = null; + while (rows.MoveNext()) + { + lastIndex = indexed.RowIndex; + lastText = rows.Current[0].GetString(); + } + Assert.Equal((2, "x"), (lastIndex, lastText)); + } + + [Theory] + [MemberData(nameof(TableBooks.Formats), MemberType = typeof(TableBooks))] + public void SheetEndClosesTheOpenTable(string format) + { + ITableBook book = TableBooks.Create(format); + book.AddSheet("Report"); + book.BeginTable("Open", ["H"]); + book.Row("a"); + book.Row("b"); + + using IExcelWorkbook workbook = Read(book.Finish()); + + Assert.Equal("A1:A3", Assert.Single(workbook.Tables).Ref); + } + + [Theory] + [MemberData(nameof(TableBooks.Formats), MemberType = typeof(TableBooks))] + public void UnstyledTableHasNoStyleName(string format) + { + ITableBook book = TableBooks.Create(format); + book.AddSheet("Report"); + book.BeginTable("Plain", ["H"], ExcelTableOptions.Default with { StyleName = null }); + book.Row("a"); + + using IExcelWorkbook workbook = Read(book.Finish()); + + Assert.Null(Assert.Single(workbook.Tables).StyleName); + } + + [Theory] + [MemberData(nameof(TableBooks.Formats), MemberType = typeof(TableBooks))] + public void FailedBeginTableWritesNothingAndClaimsNoName(string format) + { + ITableBook book = TableBooks.Create(format); + book.AddSheet("Report"); + + Assert.Throws(() => book.BeginTable("A1", ["H"])); + Assert.Throws(() => book.BeginTable("Good", ["H", "h"])); + book.BeginTable("Good", ["H"]); + book.Row("a"); + + using IExcelWorkbook workbook = Read(book.Finish()); + + Assert.Equal("A1:A2", Assert.Single(workbook.Tables).Ref); + } + + [Theory] + [MemberData(nameof(TableBooks.Formats), MemberType = typeof(TableBooks))] + public void DuplicateNameAcrossSheetsIsRejectedBeforeWriting(string format) + { + ITableBook book = TableBooks.Create(format); + book.AddSheet("One"); + book.BeginTable("Sales", ["H"]); + book.Row("a"); + book.EndSheet(); + book.AddSheet("Two"); + + Assert.Throws(() => book.BeginTable("sales", ["H"])); + book.Row("untouched"); + + using IExcelWorkbook workbook = Read(book.Finish()); + + Assert.Single(workbook.Tables); + using IExcelRowEnumerator rows = workbook.SheetAt(1).GetEnumerator(); + Assert.True(rows.MoveNext()); + Assert.Equal("untouched", rows.Current[0].GetString()); + } + + [Theory] + [MemberData(nameof(TableBooks.Formats), MemberType = typeof(TableBooks))] + public void SecondOpenTableAndEndWithoutOpenAreRejected(string format) + { + ITableBook book = TableBooks.Create(format); + book.AddSheet("Report"); + + Assert.Throws(() => book.EndTable()); + book.BeginTable("First", ["H"]); + Assert.Throws(() => book.BeginTable("Second", ["H"])); + book.Finish(); + } + + [Theory] + [MemberData(nameof(TableBooks.Formats), MemberType = typeof(TableBooks))] + public async Task AsyncBeginTableMatchesSync(string format) + { + ITableBook sync = TableBooks.Create(format); + sync.AddSheet("Report"); + sync.BeginTable("Stock", ["Product", "Qty"], ExcelTableOptions.Default with { FirstColumn = 1 }); + sync.Row(null, "Pen", 100); + + ITableBook async = TableBooks.Create(format); + async.AddSheet("Report"); + await async.BeginTableAsync("Stock", ["Product", "Qty"], ExcelTableOptions.Default with { FirstColumn = 1 }); + async.Row(null, "Pen", 100); + + Dictionary expected = TableBooks.Entries(sync.Finish()); + Dictionary actual = TableBooks.Entries(await async.FinishAsync()); + Assert.Equal(expected.Keys.Order(StringComparer.Ordinal), actual.Keys.Order(StringComparer.Ordinal)); + foreach ((string name, byte[] bytes) in expected) + { + Assert.Equal(bytes, actual[name]); + } + } + + private static IExcelWorkbook Read(byte[] package) + { + return Excel.Open(new ReadOnlyMemory(package)); + } + + private static List<(string?, int)> ParseStock(ExcelTable table) + { + List<(string?, int)> rows = []; + foreach (StockRow row in ExcelParser.FromAttributes().Parse(table.AsSheet())) + { + rows.Add((row.Product, row.Qty)); + } + return rows; + } + } +} diff --git a/tests/ExcelReader.Tests/Writer/Tables/XlsbTableShapeTests.cs b/tests/ExcelReader.Tests/Writer/Tables/XlsbTableShapeTests.cs new file mode 100644 index 00000000..433de1fd --- /dev/null +++ b/tests/ExcelReader.Tests/Writer/Tables/XlsbTableShapeTests.cs @@ -0,0 +1,130 @@ +using ExcelReader.Core.Reader.Xlsb; +using ExcelReader.Core.Writer; +using B = ExcelReader.Tests.Reader.Xlsb.Biff12Build; + +namespace ExcelReader.Tests.Writer.Tables +{ + public class XlsbTableShapeTests + { + [Fact] + public void TablePartHasExcelsRecordSequence() + { + List<(int Id, byte[] Payload)> records = Records(TableBooks.Entry(StockPackage(), "xl/tables/table1.bin")); + + Assert.Equal([343, 161, 162, 345, 347, 348, 347, 348, 346, 513, 344], records.Select(static r => r.Id)); + } + + [Fact] + public void BeginListMatchesExcelsLayout() + { + byte[] payload = Records(TableBooks.Entry(StockPackage(), "xl/tables/table1.bin"))[0].Payload; + + byte[] expected = + [ + .. B.U32(0), .. B.U32(2), .. B.U32(0), .. B.U32(1), + .. B.U32(0), .. B.U32(1), .. B.U32(1), .. B.U32(0), .. B.U32(0), + .. B.NullWideString(), .. B.NullWideString(), .. B.NullWideString(), + .. B.NullWideString(), .. B.NullWideString(), .. B.NullWideString(), + .. B.U32(0), + .. B.WideString("Stock"), .. B.WideString("Stock"), .. B.WideString(""), + .. B.NullWideString(), .. B.NullWideString(), .. B.NullWideString(), + ]; + Assert.Equal(108, payload.Length); + Assert.Equal(expected, payload); + } + + [Fact] + public void ColumnsAutoFilterAndStyleMatchExcelsLayout() + { + List<(int Id, byte[] Payload)> records = Records(TableBooks.Entry(StockPackage(), "xl/tables/table1.bin")); + + Assert.Equal([.. B.U32(0), .. B.U32(2), .. B.U32(0), .. B.U32(1)], records[1].Payload); + Assert.Equal(B.U32(2), records[3].Payload); + Assert.Equal( + [ + .. B.U32(1), .. B.U32(0), .. B.NullWideString(), .. B.NullWideString(), .. B.NullWideString(), .. B.U32(0), + .. B.NullWideString(), .. B.WideString("Product"), + .. B.NullWideString(), .. B.NullWideString(), .. B.NullWideString(), .. B.NullWideString(), + ], + records[4].Payload); + Assert.Equal([.. B.U16(4), .. B.WideString("TableStyleMedium2")], records[9].Payload); + } + + [Fact] + public void StyleFlagsAndNullStyleAreEncoded() + { + ITableBook book = TableBooks.Create("xlsb"); + book.AddSheet("S"); + book.BeginTable("Plain", ["H"], new ExcelTableOptions + { + StyleName = null, + ShowFirstColumn = true, + ShowLastColumn = true, + ShowRowStripes = false, + ShowColumnStripes = true, + }); + + byte[] style = Records(TableBooks.Entry(book.Finish(), "xl/tables/table1.bin"))[^2].Payload; + + Assert.Equal([.. B.U16(1 | 2 | 8), .. B.NullWideString()], style); + } + + [Fact] + public void SheetListsItsTablePartsBeforeEndSheet() + { + byte[] package = StockPackage(); + + List<(int Id, byte[] Payload)> sheet = Records(TableBooks.Entry(package, "xl/worksheets/sheet1.bin")); + + Assert.Equal([660, 661, 662, 130], sheet.Skip(sheet.Count - 4).Select(static r => r.Id)); + Assert.Equal(B.U32(1), sheet[^4].Payload); + Assert.Equal(B.WideString("rId1"), sheet[^3].Payload); + string types = System.Text.Encoding.UTF8.GetString(TableBooks.Entry(package, "[Content_Types].xml")); + Assert.Contains("", types, StringComparison.Ordinal); + Assert.Contains("Target=\"../tables/table1.bin\"", System.Text.Encoding.UTF8.GetString(TableBooks.Entry(package, "xl/worksheets/_rels/sheet1.bin.rels")), StringComparison.Ordinal); + } + + [Fact] + public void SheetWithoutTableHasNoAutoFilterOrSortState() + { + ITableBook book = TableBooks.Create("xlsb"); + book.AddSheet("S"); + book.Row("Pen", 100); + + List ids = [.. Records(TableBooks.Entry(book.Finish(), "xl/worksheets/sheet1.bin")).Select(static r => r.Id)]; + + Assert.DoesNotContain(161, ids); + Assert.DoesNotContain(648, ids); + Assert.Equal([146, 130], ids.Skip(ids.Count - 2)); + } + + [Fact] + public void TableSheetTailAfterSheetDataIsOnlyListParts() + { + List ids = [.. Records(TableBooks.Entry(StockPackage(), "xl/worksheets/sheet1.bin")).Select(static r => r.Id)]; + + Assert.Equal([660, 661, 662, 130], ids.Skip(ids.IndexOf(146) + 1)); + } + + private static byte[] StockPackage() + { + ITableBook book = TableBooks.Create("xlsb"); + book.AddSheet("S"); + book.BeginTable("Stock", ["Product", "Qty"]); + book.Row("Pen", 100); + book.Row("Notebook", 40); + return book.Finish(); + } + + private static List<(int Id, byte[] Payload)> Records(byte[] part) + { + List<(int, byte[])> records = []; + Biff12RecordReader reader = new(part); + while (reader.TryReadRecord(out int id, out ReadOnlySpan payload)) + { + records.Add((id, payload.ToArray())); + } + return records; + } + } +} diff --git a/tests/ExcelReader.Tests/Writer/Tables/XlsxTableShapeTests.cs b/tests/ExcelReader.Tests/Writer/Tables/XlsxTableShapeTests.cs new file mode 100644 index 00000000..56f6de6f --- /dev/null +++ b/tests/ExcelReader.Tests/Writer/Tables/XlsxTableShapeTests.cs @@ -0,0 +1,104 @@ +using System.Text; +using ExcelReader.Core.Writer; + +namespace ExcelReader.Tests.Writer.Tables +{ + public class XlsxTableShapeTests + { + [Fact] + public void SheetReferencesItsTablesAndDeclaresTheRelationshipsNamespace() + { + byte[] package = TwoTables(); + + string sheet = Text(package, "xl/worksheets/sheet1.xml"); + + Assert.StartsWith( + "", + sheet, StringComparison.Ordinal); + Assert.EndsWith( + "", + sheet, StringComparison.Ordinal); + } + + [Fact] + public void SheetRelsAndContentTypesListTheTables() + { + byte[] package = TwoTables(); + + Assert.Equal( + "" + + "" + + "" + + "", + Text(package, "xl/worksheets/_rels/sheet1.xml.rels")); + string types = Text(package, "[Content_Types].xml"); + Assert.Contains("", types, StringComparison.Ordinal); + Assert.Contains("", types, StringComparison.Ordinal); + } + + [Fact] + public void TablePartCarriesNameRangeColumnsAndStyleFlags() + { + byte[] package = TwoTables(); + + Assert.Equal( + "" + + "" + + "" + + "" + + "" + + "
", + Text(package, "xl/tables/table2.xml")); + } + + [Fact] + public void UnstyledTableOmitsTheStyleName() + { + ITableBook book = TableBooks.Create("xlsx"); + book.AddSheet("S"); + book.BeginTable("Plain", ["H"], ExcelTableOptions.Default with { StyleName = null }); + + string table = Text(book.Finish(), "xl/tables/table1.xml"); + + Assert.Contains("", table, StringComparison.Ordinal); + } + + [Fact] + public void ColumnNameWithAnEscapeLookalikeIsEscapedLikeItsHeaderCell() + { + ITableBook book = TableBooks.Create("xlsx"); + book.AddSheet("S"); + book.BeginTable("Codes", ["Code_x0041_"]); + book.Row("a"); + byte[] package = book.Finish(); + + Assert.Contains("", Text(package, "xl/tables/table1.xml"), StringComparison.Ordinal); + } + + private static byte[] TwoTables() + { + ITableBook book = TableBooks.Create("xlsx"); + book.AddSheet("S"); + book.BeginTable("Stock", ["Product"]); + book.Row("Pen"); + book.EndTable(); + book.Row(); + book.Row(); + book.BeginTable("Prices", ["Item", "Qty & \"Price\""], new ExcelTableOptions + { + FirstColumn = 1, + StyleName = "TableStyleLight9", + ShowRowStripes = false, + ShowColumnStripes = true, + ShowFirstColumn = true, + }); + book.Row(null, "Pen", 2); + return book.Finish(); + } + + private static string Text(byte[] package, string name) + { + return Encoding.UTF8.GetString(TableBooks.Entry(package, name)); + } + } +} diff --git a/tests/ExcelReader.Tests/Writer/WorkbookWriterTests.cs b/tests/ExcelReader.Tests/Writer/WorkbookWriterTests.cs index af1a9491..1c575393 100644 --- a/tests/ExcelReader.Tests/Writer/WorkbookWriterTests.cs +++ b/tests/ExcelReader.Tests/Writer/WorkbookWriterTests.cs @@ -12,6 +12,36 @@ namespace ExcelReader.Tests.Writer { public class WorkbookWriterTests { + [Fact] + public async Task XlsxDate1904RoundTrips() + { + CancellationToken ct = TestContext.Current.CancellationToken; + var date = new DateTime(1904, 1, 2, 6, 0, 0, DateTimeKind.Unspecified); + var day = new DateOnly(2024, 5, 6); + var ms = new MemoryStream(); + await using (XlsxWorkbookWriter wb = XlsxWorkbookWriter.Create(ms, leaveOpen: true, new XlsxWriterOptions { Date1904 = true })) + { + XlsxSheetWriter sheet = wb.AddSheet("S"); + await using (XlsxRowWriter row = await sheet.StartRowAsync(ct)) + { + row.Write(date); + row.Write(day); + } + await sheet.EndAsync(ct); + await wb.EndAsync(ct); + } + ms.Position = 0; + + using XlsxWorkbook reader = Excel.FromXlsx(ms); + Assert.True(reader.IsDate1904); + using XlsxWorkbook.Enumerator e = reader.FirstSheet.GetEnumerator(); + Assert.True(e.MoveNext()); + Assert.True(e.Current[0].TryGetDateTime(isDate1904: true, out DateTime parsed)); + Assert.Equal(date, parsed); + Assert.True(e.Current[1].TryGetDateTime(isDate1904: true, out DateTime parsedDay)); + Assert.Equal(day.ToDateTime(TimeOnly.MinValue), parsedDay); + } + private sealed class StringRow { @@ -114,9 +144,9 @@ private sealed class IgnoreRow private sealed class MoneyConverter : IExcelCellConverter, IExcelCellWriter { - public bool TryConvert(in Cell cell, bool isDate1904, IFormatProvider provider, out Money value) + public bool TryConvert(in Cell cell, in ExcelRowContext context, out Money value) { - if (cell.TryParse(provider, out decimal amount)) + if (cell.TryParse(context.FormatProvider, out decimal amount)) { value = new Money(amount); return true; @@ -833,7 +863,7 @@ public async Task HeaderRowTwoConfig() }).ConfigureAwait(true); await using var reader = Excel.FromXlsx(ms); - var config = new ExcelParserConfig { HeaderRow = 2 }; + var config = new ExcelParserOptions { HeaderRow = 2 }; var rows = ExcelParser.FromAttributes(config).Parse(reader.FirstSheet).ToList(); Assert.Single(rows); Assert.Equal("HeaderRow2", rows[0].Name); diff --git a/tests/ExcelReader.Tests/Writer/Xls/XlsWriterTests.cs b/tests/ExcelReader.Tests/Writer/Xls/XlsWriterTests.cs index acdd2aab..7fa3c62d 100644 --- a/tests/ExcelReader.Tests/Writer/Xls/XlsWriterTests.cs +++ b/tests/ExcelReader.Tests/Writer/Xls/XlsWriterTests.cs @@ -13,7 +13,7 @@ private static async Task WriteAsync( Action build, bool date1904 = false, CancellationToken ct = default) { var ms = new MemoryStream(); - await using (var wb = XlsWorkbookWriter.Create(ms, leaveOpen: true, date1904: date1904)) + await using (var wb = XlsWorkbookWriter.Create(ms, leaveOpen: true, new XlsWriterOptions { Date1904 = date1904 })) { build(wb); await wb.EndAsync(ct); @@ -51,7 +51,7 @@ public async Task RoundTripsAllCellTypes() Assert.Equal(CellType.Boolean, e.Current[3].Type); Assert.Equal("1", e.Current[3].GetString()); Assert.Equal(CellType.Date, e.Current[4].Type); - Assert.True(e.Current[4].TryGetDateTime(out DateTime parsed)); + Assert.True(e.Current[4].TryGetDateTime(isDate1904: false, out DateTime parsed)); Assert.Equal(date, parsed); Assert.False(e.MoveNext()); } @@ -130,7 +130,7 @@ public async Task NullableAndGenericOverloadsRoundTrip() Assert.True(e.MoveNext()); Assert.Equal(CellType.Boolean, e.Current[0].Type); Assert.Equal(CellType.Empty, e.Current[1].Type); - Assert.True(e.Current[2].TryGetDateTime(out DateTime parsed)); + Assert.True(e.Current[2].TryGetDateTime(isDate1904: false, out DateTime parsed)); Assert.Equal(date, parsed); Assert.Equal(CellType.Empty, e.Current[3].Type); Assert.True(e.Current[4].TryParse(Inv, out double nullableFloat)); diff --git a/tests/ExcelReader.Tests/Writer/Xlsb/XlsbWriterTests.cs b/tests/ExcelReader.Tests/Writer/Xlsb/XlsbWriterTests.cs index dc4fd2c4..1e21e32c 100644 --- a/tests/ExcelReader.Tests/Writer/Xlsb/XlsbWriterTests.cs +++ b/tests/ExcelReader.Tests/Writer/Xlsb/XlsbWriterTests.cs @@ -205,7 +205,7 @@ public async Task NullableAndGenericOverloadsRoundTrip() Assert.True(e.MoveNext()); Assert.Equal(CellType.Boolean, e.Current[0].Type); Assert.Equal(CellType.Empty, e.Current[1].Type); - Assert.True(e.Current[2].TryGetDateTime(out DateTime parsed)); + Assert.True(e.Current[2].TryGetDateTime(isDate1904: false, out DateTime parsed)); Assert.Equal(date, parsed); Assert.Equal(CellType.Empty, e.Current[3].Type); Assert.True(e.Current[4].TryParse(null, out double nullableFloat)); diff --git a/tests/ExcelReader.Tests/data/tables/tables.xlsb b/tests/ExcelReader.Tests/data/tables/tables.xlsb new file mode 100644 index 00000000..de4e2719 Binary files /dev/null and b/tests/ExcelReader.Tests/data/tables/tables.xlsb differ diff --git a/tests/ExcelReader.Tests/data/tables/tables.xlsx b/tests/ExcelReader.Tests/data/tables/tables.xlsx new file mode 100644 index 00000000..a2452dbf Binary files /dev/null and b/tests/ExcelReader.Tests/data/tables/tables.xlsx differ