From 84f3a14206b1160a486e6910430a8b8d1d48195b Mon Sep 17 00:00:00 2001 From: bbalouki Date: Wed, 24 Jun 2026 14:59:08 +0100 Subject: [PATCH 01/58] Harden core (IMPROVEMENTS Phase 0) and expand test suite (Phase 1) Phase 0 - correctness & robustness in soatable.hpp: - 0.1 Replace signed sparse index (int32 + tombstone=-1) with an unsigned size_type alias and npos sentinel; remove all size_t round-trip casts. - 0.2 Drop the unused include from the public header. - 0.3 Document per-method exception guarantees; make emplace (new value) and insert (grow path) strong by reserving/committing so a throw cannot desync the parallel arrays; reorder rebuilds indices then swaps. - 0.4 Retire a slot instead of recycling it when its generation saturates, so a stale handle can never alias a freshly inserted row (ABA wraparound). - 0.5 Gate the sstd namespace alias behind SOATABLE_ENABLE_SSTD_ALIAS. Phase 1 - testing & quality gates: - Restructure tests into tests/unittests/ (7 -> 47 runtime tests). - Add move_only and throws_on_nth column types; prove strong guarantee. - Free-list churn, ABA stale-handle, large-table, select edge cases, const-correctness, row_handle and compression round-trip coverage. - Fixed-seed invariant/property test over 3000 randomized ops. - Compile-fail try_compile contracts and an include-hygiene TU test. - Strict-warning gate target and an ASan+UBSan CI leg. Verified: 50/50 ctest green; header clean under clang -Wconversion -Wsign-conversion -Werror and MSVC /W4 /WX; ASan+UBSan clean under churn. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/ci.yaml | 15 ++ .gitignore | 1 + CLAUDE.md | 101 +++++++ FEATURES.md | 223 ++++++++++++++++ IMPROVEMENTS.md | 264 +++++++++++++++++++ include/soatable/soatable.hpp | 111 +++++--- tests/CMakeLists.txt | 59 ++++- tests/compile_fail/duplicate_columns.cpp | 15 ++ tests/compile_fail/empty_table.cpp | 9 + tests/compile_fail/unregistered_column.cpp | 20 ++ tests/soatable_tests.cpp | 159 ----------- tests/unittests/batch_test.cpp | 53 ++++ tests/unittests/compression_test.cpp | 105 ++++++++ tests/unittests/exception_safety_test.cpp | 75 ++++++ tests/unittests/include_hygiene_other_tu.cpp | 19 ++ tests/unittests/include_hygiene_test.cpp | 31 +++ tests/unittests/invariant_test.cpp | 93 +++++++ tests/unittests/row_handle_test.cpp | 59 +++++ tests/unittests/select_test.cpp | 126 +++++++++ tests/unittests/sort_test.cpp | 113 ++++++++ tests/unittests/table_test.cpp | 178 +++++++++++++ tests/unittests/test_types.hpp | 73 +++++ tests/warning_check.cpp | 33 +++ 23 files changed, 1743 insertions(+), 192 deletions(-) create mode 100644 CLAUDE.md create mode 100644 FEATURES.md create mode 100644 IMPROVEMENTS.md create mode 100644 tests/compile_fail/duplicate_columns.cpp create mode 100644 tests/compile_fail/empty_table.cpp create mode 100644 tests/compile_fail/unregistered_column.cpp delete mode 100644 tests/soatable_tests.cpp create mode 100644 tests/unittests/batch_test.cpp create mode 100644 tests/unittests/compression_test.cpp create mode 100644 tests/unittests/exception_safety_test.cpp create mode 100644 tests/unittests/include_hygiene_other_tu.cpp create mode 100644 tests/unittests/include_hygiene_test.cpp create mode 100644 tests/unittests/invariant_test.cpp create mode 100644 tests/unittests/row_handle_test.cpp create mode 100644 tests/unittests/select_test.cpp create mode 100644 tests/unittests/sort_test.cpp create mode 100644 tests/unittests/table_test.cpp create mode 100644 tests/unittests/test_types.hpp create mode 100644 tests/warning_check.cpp diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5a1d194..bcb6feb 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -24,6 +24,12 @@ jobs: os: ubuntu-latest, cxx: clang++-18 } + - { + name: "Ubuntu Clang 18 (ASan+UBSan)", + os: ubuntu-latest, + cxx: clang++-18, + sanitizers: true + } - { name: "Windows MSVC", os: windows-latest, @@ -50,6 +56,12 @@ jobs: run: | if [ "${{ matrix.config.os }}" = "windows-latest" ]; then cmake -B build -DSOATABLE_BUILD_TESTS=ON -DSOATABLE_BUILD_EXAMPLES=ON + elif [ "${{ matrix.config.sanitizers }}" = "true" ]; then + # AddressSanitizer + UndefinedBehaviorSanitizer; -fno-sanitize-recover makes UBSan abort + # so any finding fails the build. Benchmarks are excluded to keep the leg focused. + cmake -B build -DSOATABLE_BUILD_TESTS=ON -DSOATABLE_BUILD_EXAMPLES=ON -DSOATABLE_BUILD_BENCHMARKS=OFF \ + -DCMAKE_CXX_COMPILER=${{ matrix.config.cxx }} \ + -DCMAKE_CXX_FLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -fno-omit-frame-pointer" else cmake -B build -DSOATABLE_BUILD_TESTS=ON -DSOATABLE_BUILD_EXAMPLES=ON -DCMAKE_CXX_COMPILER=${{ matrix.config.cxx }} fi @@ -61,5 +73,8 @@ jobs: - name: Run Tests run: | cd build + # Treat any sanitizer finding as a hard failure. + export ASAN_OPTIONS=halt_on_error=1:abort_on_error=1 + export UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 ctest -C Release --output-on-failure shell: bash diff --git a/.gitignore b/.gitignore index 3a6474d..01a2aa4 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ install/ CMakeUserPresets.json *-workspace /.vscode/*.json +prompts.txt diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..659db26 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,101 @@ +# General Best Practices + +## Testing Philosophy + +- Use real implementations; only mock external dependencies (LLM APIs, cloud services). +- Test public interface behavior, not implementation details — keeps tests resilient to refactoring. +- Tests must be fast, isolated, and have descriptive names; cover edge cases and error conditions. +- Location: `tests/unittests/` following source structure. + +## Comments + +- Explain the _why_, not the _what_. Well-named code is self-documenting. +- Write comments as complete sentences; block comments begin with `# ` (space after hash). + +## Versioning & Breaking Changes + +- Follow Semantic Versioning 2.0.0 (`MAJOR.MINOR.PATCH`). +- A breaking change is any backward-incompatible modification to the public API, including data schemas, CLI, and server communication formats. + +--- + +# Modern C++ Best Practices + +## Language & Compiler + +- Target C++17 minimum; prefer C++20/23. Set `CMAKE_CXX_STANDARD` explicitly; disable extensions (`-pedantic`). +- Enable `-Wall -Wextra -Wconversion -Wsign-conversion` and treat warnings as errors (`-Werror`). + +## Style & Naming + +- `m_` prefix for private members; `snake_case` for variables/functions; `PascalCase` for types; `SCREAMING_SNAKE_CASE` for macros and Constants. +- All symbols must be at least 3 characters long. Use `std::print`/`std::println`/`std::format` for output. +- No magic literals — replace with named `constexpr` values. Declare variables close to first use. +- Keep functions short and flat; remove unused variables, parameters, and includes. +- Use `#pragma once`; keep headers to declarations and inline/template definitions only. +- Do not use mdash (`--`) in code, comments, or documentation. + +## Variables & Types + +- Always initialize variables at declaration; prefer brace initialization `T x{value}` to prevent narrowing. +- Use `auto` when type is obvious; `const` by default; `constexpr` for compile-time constants. +- Avoid `using namespace std` in headers. Use `enum class`; use `std::string_view` for read-only strings. +- Never mix signed/unsigned arithmetic; never use C-style casts — use `static_cast`, `dynamic_cast`, etc. +- Never use raw pointers for ownership; use references or smart pointers instead. Avoid `void*` and `reinterpret_cast`. +- Never use `goto` or `longjmp`; prefer structured control flow and RAII for cleanup. +- Never use C-style arrays or C-style strings; prefer `std::array`, `std::vector`, and `std::string` for safety and convenience. + +## Memory Management + +- Follow RAII: tie every resource (memory, file, mutex) to an object's lifetime. +- Use `std::unique_ptr`/`std::shared_ptr`/`std::weak_ptr` via `make_unique`/`make_shared` — never `new`/`delete` directly. +- Follow Rule of Zero (all RAII members) or Rule of Five (when owning a raw resource). +- Detect leaks with AddressSanitizer; use standard containers over raw arrays. + +## Functions & Classes + +- One logical operation per function; always use trailing return types. +- Pass non-trivial types as `const T&`; return by value (rely on NRVO/RVO). +- Mark `[[nodiscard]]` where return values must not be ignored; mark `noexcept` when applicable. +- Single-argument constructors must be `explicit`; keep members `private`; use `override`/`final` on overrides. +- Prefer composition over inheritance; declare base destructors `virtual` (or `protected` non-virtual). +- Do not call virtual functions from constructors or destructors. + +## Error Handling + +- Use exceptions for errors (not for normal control flow); catch by `const` reference to avoid slicing. +- Maintain a consistent exception-safety guarantee: basic, strong, or no-throw. +- Use `static_assert` for compile-time invariants; `assert()` for debug-only preconditions — never on user input. +- Consider `std::expected` (C++23) for predictable, performance-sensitive failure paths. + +## Templates & Concurrency + +- Constrain templates with C++20 concepts; use `if constexpr` over SFINAE/`enable_if`. +- Use variadic templates and fold expressions instead of recursive specializations. +- Protect shared mutable state with `std::mutex`; prefer `std::jthread` over `std::thread`. +- Use `std::atomic` for lock-free single-variable operations; never `volatile` for inter-thread communication. +- Minimize lock duration; avoid callbacks or virtual calls while holding a lock; prefer `std::async` and parallel algorithms. + +## Performance & Build + +- Profile before optimizing; prefer `std::vector` for cache-friendly access; use `reserve()` when size is known. +- Use move semantics to avoid deep copies; prefer standard algorithms (`std::sort`, `std::transform`) over hand-written loops. +- Use CMake (≥ 3.25) with target-centric paradigm: `target_include_directories`, `target_compile_options`, `target_link_libraries`. +- Use `FetchContent` or vcpkg/Conan for dependencies; use out-of-source builds and CMake Presets. +- Enable `CMAKE_EXPORT_COMPILE_COMMANDS=ON` for tooling; use `ccache`/`sccache` to speed up rebuilds. + +## Testing & Tooling + +- Use GoogleTest/Catch2 with CTest; write tests before or alongside implementation. +- Run tests under AddressSanitizer and UndefinedBehaviourSanitizer; use fuzz testing for parsers and untrusted input. +- Enforce clang-format in CI; use clang-tidy with `cppcoreguidelines-*`, `modernize-*`, `readability-*` checks. +- Use debuggers (GDB, LLDB) actively rather than `std::cout`; use `compile_commands.json` for accurate tooling. + +## CI/CD & Maintenance + +- CI must build and test before merging; build across GCC, Clang, and MSVC. +- Document the _why_, not the _what_; use Doxygen (`///`) for API docs and `//` for implementation notes. +- Keep public API surface minimal; mark deprecated APIs with `[[deprecated("reason")]]` and include a migration path. +- Follow the Boy Scout Rule; track technical debt explicitly; maintain a `CHANGELOG.md` and `CONTRIBUTING.md`. + +--- diff --git a/FEATURES.md b/FEATURES.md new file mode 100644 index 0000000..7759e03 --- /dev/null +++ b/FEATURES.md @@ -0,0 +1,223 @@ +# FEATURES.md + +> The **net-new capabilities** that would take SoaTable from a strong sparse-column container to +> _"the numpy of Structure-of-Arrays"_ — portable, domain-agnostic, and best-in-class. For +> hardening of what already exists, see [IMPROVEMENTS.md](IMPROVEMENTS.md). + +## Vision + +numpy won by being three things at once: a **cache-friendly contiguous array**, a **rich compute +layer** over that array, and a **universal interchange format** every language and tool speaks. +SoaTable already nails the _layout_ primitive (sparse columns + generational handles). To become the +numpy of SoA it needs the **compute layer**, the **interchange story**, and the **portability** that +let finance, engineering, aerospace, games, and data science all reach for it. + +Each feature is framed as **What / Use cases / Sketch / Priority / Effort**, where priority is **P0** +(foundational), **P1** (high value), **P2** (differentiator), and effort is **S / M / L**. + +Bar-setting references: [soagen](https://marzer.github.io/soagen/) (SIMD alignment, contiguous +allocation, tuple rows, swizzling, binary serialization, `.natvis`), +[Apache Arrow](https://arrow.apache.org/docs/format/Columnar.html) (columnar interchange, 64-byte +alignment, null bitmaps), [EnTT](https://github.com/skypjack/entt) (sparse-set ECS), AoSoA hybrid +layouts, and emerging **C++26 reflection**-based SoA generation. + +--- + +## Roadmap at a glance + +| Phase | Theme | Priority | Target | +| ----- | --------------------------------------- | -------- | ------ | +| A | Foundational parity (layout primitives) | P0 / P1 | v0.5 | +| B | Compute layer ("the numpy part") | P1 | v0.7 | +| C | Portability & interop | P1 / P2 | v0.8 | +| D | Scale & concurrency | P2 | v0.9 | +| E | Domain enablers & developer experience | P2 | v1.0 | + +--- + +## Phase A — Foundational parity with best-in-class (P0 / P1) + +### A.1 Zero-copy column spans + +- **What:** A `column() -> std::span` (and `const` form) exposing a column's dense data as a + contiguous, mutable view. +- **Use cases:** Hand a column straight to BLAS/LAPACK (finance risk matrices), an FFT (engineering + signal processing), or a SIMD kernel — no copy. This _is_ the "numpy array" primitive. +- **Sketch:** Return a span over the existing dense `m_data`; pair with a `row_ids()` span so callers + can map dense positions back to handles. Document that erase/sort can invalidate spans. +- **Priority:** P0 · **Effort:** S + +### A.2 SIMD-aligned columns and optional AoSoA layout + +- **What:** Per-column over-alignment (32 / 64 bytes) plus an opt-in tiled **AoSoA** storage mode. +- **Use cases:** Vectorized physics/particle updates (aerospace, games), columnar analytics. Cited + layout-driven gains of 40–60% on real workloads; AoSoA gets SoA throughput with better cache + locality. +- **Sketch:** An aligned allocator behind the column vectors + `std::assume_aligned` in hot loops; a + policy template selecting `SoA` vs `AoSoA` so the API is unchanged. Aligns with Arrow's + 64-byte recommendation. +- **Priority:** P0 · **Effort:** L + +### A.3 First-class null / validity bitmaps + +- **What:** Surface per-column presence as an Arrow-style validity bitmap, complementing the existing + per-row signature. +- **Use cases:** Sparse financial panels, sensor logs with dropouts, any "mostly-empty" column — and a + prerequisite for Arrow interchange (A.4). +- **Sketch:** Expose `validity() -> bitmap_view`; let bulk ops consume it for branchless masked + iteration. **Priority:** P1 · **Effort:** M + +### A.4 Serialization & persistence + +- **What:** (1) A trivially-copyable fast path that dumps/loads columns as raw bytes (like soagen's + `data()`/size), and (2) a portable, **versioned** on-disk format. (3) An optional Arrow / Parquet + bridge. +- **Use cases:** Snapshot a simulation (aerospace), persist a tick store (finance), move tables between + C++/Python/Rust/Spark without re-encoding (Arrow is the lingua franca of analytics). +- **Sketch:** `static_assert(std::is_trivially_copyable_v)`-gated fast path; a schema header carrying + version + column types for the portable path; a thin adapter mapping columns to Arrow `Array`s behind + an optional dependency. **Priority:** P1 · **Effort:** L + +--- + +## Phase B — Compute layer, "the numpy part" (P1) + +### B.1 Vectorized column ops (ufuncs) + +- **What:** `transform`, `reduce`, `scan`, `filter`, and element-wise arithmetic over columns, with a + `std::execution` / SIMD backend. +- **Use cases:** `pnl = price * qty` across a million rows (finance); `magnitude = sqrt(vx²+vy²+vz²)` + (engineering) — expressed once, vectorized automatically. +- **Sketch:** Free functions over `std::span` columns (A.1) dispatching to parallel/SIMD algorithms; + optionally fuse element-wise chains. **Priority:** P1 · **Effort:** L + +### B.2 Query / expression DSL + +- **What:** Composable predicates and projections layered on `select<>()`, with filter push-down and + groupby / aggregate primitives. +- **Use cases:** `table.where(price > 100 && active).select()`; group telemetry by region + and average — analytics ergonomics without leaving C++. +- **Sketch:** Lightweight expression templates compiling to the existing smallest-driver scan plus a + predicate; `group_by().agg(sum())` building on B.1. **Priority:** P1 · **Effort:** L + +### B.3 Masked & strided views, broadcasting + +- **What:** Views that iterate a subset (a mask or stride) of a column and broadcast scalars over a + column where it reads naturally. +- **Use cases:** Apply a correction to flagged rows only; add a scalar bias to a whole column. +- **Sketch:** A `masked_span` / `strided_span` adapter feeding the B.1 ops. **Priority:** P2 · **Effort:** M + +--- + +## Phase C — Portability & interop (P1 / P2) + +### C.1 C API + stable-ABI shim and language bindings + +- **What:** A C-linkage facade over a fixed set of column types, plus first-class **Python** bindings + (nanobind/pybind11) and a path for Rust / C# FFI. +- **Use cases:** This is the literal "numpy" forcing function — let data scientists `import soatable` + and get zero-copy numpy arrays via the buffer protocol; let Rust/C# embed the same store. +- **Sketch:** A C ABI exposing handle/column/select; a Python module mapping columns to numpy arrays + through A.1 spans + the buffer protocol. **Priority:** P1 · **Effort:** L + +### C.2 Custom allocators + +- **What:** Allocator-aware columns: `std::pmr`, arena/bump, and hooks for GPU/pinned memory. +- **Use cases:** Deterministic, fragmentation-free allocation for real-time/aerospace; arena reuse in + game frames; staging buffers for GPU offload. +- **Sketch:** Thread an allocator template/parameter through `ColumnVector`; default unchanged. + **Priority:** P1 · **Effort:** M + +### C.3 Freestanding / no-exceptions subset + +- **What:** A compile mode with no exceptions, no RTTI, and minimal heavy includes (ties to + [IMPROVEMENTS.md](IMPROVEMENTS.md) 0.2), surfacing `std::expected`-style error returns. +- **Use cases:** Embedded flight software, microcontrollers, kernel/driver contexts where the STL + exception machinery is banned. +- **Sketch:** Guard exception paths behind `SOATABLE_NO_EXCEPTIONS`; provide error-code/`expected` + accessors. **Priority:** P2 · **Effort:** M + +### C.4 C++26 reflection path + +- **What:** When the toolchain supports it, generate columns directly from a plain `struct` — no manual + type list — while keeping the explicit type-list API as the portable fallback. +- **Use cases:** `SoaTable` from `struct Particle { float x, y, z; ... };` with zero + boilerplate, matching the ergonomics modern users expect. +- **Sketch:** A reflection adapter enumerating members into columns behind a feature test. + **Priority:** P2 · **Effort:** L + +--- + +## Phase D — Scale & concurrency (P2) + +### D.1 Concurrent / thread-safe table + +- **What:** A variant supporting concurrent readers with safe writers — sharded storage and/or + snapshot (MVCC) read views. +- **Use cases:** A market-data table read by many strategy threads while one thread ingests; an ECS + world iterated by parallel systems. +- **Sketch:** Reader-friendly snapshots with copy-on-write column versions, or per-shard locking with a + stable global handle space. **Priority:** P2 · **Effort:** L + +### D.2 Chunked column storage + +- **What:** Store each column as a sequence of fixed-size chunks (Arrow-style record batches) instead of + one monolithic vector. +- **Use cases:** Bounds reallocation/copy cost on growth, enables streaming and parallelism by chunk, + and is the natural unit for Arrow interchange (A.4). +- **Sketch:** A `chunked_column` with the same span/iteration surface; `select` iterates chunk by + chunk. **Priority:** P2 · **Effort:** L + +### D.3 Out-of-core / memory-mapped columns + +- **What:** Back columns with `mmap`ed files so datasets exceed RAM. +- **Use cases:** Multi-billion-row historical tick stores; large engineering datasets on workstations. +- **Sketch:** A memory-mapped allocator (composes with C.2) for trivially-copyable columns; lazy + paging. **Priority:** P2 · **Effort:** L + +--- + +## Phase E — Domain enablers & developer experience (P2) + +### E.1 Domain cookbooks & ready-made examples + +- **What:** End-to-end examples proving "literally anyone can use it": finance (tick / portfolio + tables), ECS/game world, aerospace telemetry, scientific data table. +- **Use cases:** Adoption — concrete recipes turn curiosity into usage faster than reference docs. +- **Sketch:** One worked example per domain under `example/`, each linked from the README. + **Priority:** P2 · **Effort:** M + +### E.2 Debugger visualizers + +- **What:** MSVC `.natvis` and GDB/LLDB pretty-printers that render a table as rows×columns. +- **Use cases:** Inspecting a sparse table in a debugger is otherwise painful; soagen ships `.natvis` + for exactly this. **Priority:** P2 · **Effort:** S + +### E.3 Runtime schema evolution & typed metadata + +- **What:** Add/remove columns at runtime, attach typed metadata, and carry **units** on columns. +- **Use cases:** Evolving schemas without recompiling; dimensional-safety for engineering (reject + adding metres to seconds at the boundary). +- **Sketch:** A type-erased dynamic-column layer alongside the static one; optional units tags checked + at op boundaries. **Priority:** P2 · **Effort:** L + +### E.4 Time-series helpers + +- **What:** Higher-level helpers (rolling windows, deltas, dirty-region scans) built on the existing + `DeltaValue` and `DirtyMask`. +- **Use cases:** Smooth telemetry tracking (aerospace), incremental recompute of changed rows (games, + finance). **Priority:** P2 · **Effort:** M + +--- + +## Sequencing notes + +- **A.1 (spans)** is the keystone: B (compute), C.1 (Python/numpy), and A.4 (serialization) all build + on contiguous column views, so land it first. +- **A.4 + D.2** together unlock the Arrow interchange story that makes SoaTable a true ecosystem + citizen. +- **C.1 (Python bindings)** is the single highest-leverage adoption lever toward the "numpy of SoA" + identity and should be prioritized once A.1/B.1 exist. +- Every feature here assumes the Phase 0/1 correctness and test work in + [IMPROVEMENTS.md](IMPROVEMENTS.md) is in place first — new surface area on an unverified core just + multiplies risk. diff --git a/IMPROVEMENTS.md b/IMPROVEMENTS.md new file mode 100644 index 0000000..46911db --- /dev/null +++ b/IMPROVEMENTS.md @@ -0,0 +1,264 @@ +# IMPROVEMENTS.md + +> A clear-eyed account of what should be **fixed, hardened, and strengthened** in the +> _existing_ SoaTable implementation. For _net-new_ capabilities, see [FEATURES.md](FEATURES.md). + +SoaTable (v0.1.0) is a header-only C++23 Structure-of-Arrays (SoA) table. The core design is +strong: a sparse 3-vector [`ColumnVector`](include/soatable/soatable.hpp) (sparse → dense → data), +generational [`row_id`](include/soatable/soatable.hpp) handles that defeat the ABA problem, an +optimized `select<>()` join that drives iteration off the smallest required column, batch ops, +single / multi / parallel sorting, and compression helpers (`quantized_float`, `packed_bits`, +`DeltaValue`, `DirtyMask`). + +The goal is to make SoaTable _"the numpy of SoA"_: portable, domain-agnostic (finance, +engineering, aerospace), and best-in-class. This document is the **hardening roadmap** to get the +existing foundation to production grade. Items are framed as **What / Why / Approach / Priority / +Effort**, where priority is **P0** (correctness/blocker), **P1** (important), **P2** (polish), and +effort is **S / M / L**. + +Benchmarked against the landscape that defines "top tier": [soagen](https://marzer.github.io/soagen/) +(single contiguous allocation, per-column SIMD alignment, tuple-protocol rows, `.natvis`), +[Apache Arrow](https://arrow.apache.org/docs/format/Columnar.html) (columnar format, 64-byte +alignment, null bitmaps), [EnTT](https://github.com/skypjack/entt) (sparse-set ECS), and AoSoA +hybrid layouts. + +--- + +## Roadmap at a glance + +| Phase | Theme | Priority | Target | +| ----- | ---------------------------------- | -------- | ------ | +| 0 | Correctness & robustness | P0 | v0.2 | +| 1 | Testing & quality gates | P0 / P1 | v0.2 | +| 2 | API ergonomics & const-correctness | P1 | v0.3 | +| 3 | Build, packaging & docs | P1 / P2 | v0.3 | +| 4 | Performance hardening | P2 | v0.4 | + +--- + +## Phase 0 — Correctness & robustness (P0) + +### 0.1 Index width and signed/unsigned mixing + +- **What:** `ColumnVector::m_sparse` is `std::vector` with `tombstone = -1` + ([soatable.hpp:396-398](include/soatable/soatable.hpp#L396-L398)). Every access casts back to + `std::size_t` (e.g. [:444](include/soatable/soatable.hpp#L444), + [:490](include/soatable/soatable.hpp#L490)). +- **Why:** Signed dense indices cap a single column at ~2.1B rows and mix signed/unsigned + arithmetic, which the project style guide (CLAUDE.md) explicitly forbids and which + `-Wsign-conversion` should be flagging. Silent narrowing on huge tables is a latent corruption + bug. +- **Approach:** Use an unsigned dense index (`std::uint32_t` or a `std::size_t` typedef) plus an + explicit `static constexpr ... npos = (max)()` sentinel instead of `-1`. Introduce a single + `using size_type = ...;` alias so a future 64-bit build is a one-line change. Keep `row_id` + fields as documented public ABI. +- **Priority:** P0 · **Effort:** M + +### 0.2 Remove `` from the public header + +- **What:** [soatable.hpp:12](include/soatable/soatable.hpp#L12) includes `` even though + the library has a `` fast path ([:22-27](include/soatable/soatable.hpp#L22-L27)). +- **Why:** `` injects a static `std::ios_base::Init` object into _every_ translation unit + that includes the header, inflating binary size and startup cost — unacceptable for a header-only + library meant to be embedded "everywhere" (including freestanding/aerospace targets). +- **Approach:** Drop the include if unused; if any streaming helper needs it, move that helper to an + opt-in `` guarded behind a macro. Audit the remaining heavy includes + (``, ``) for the same "pay only if you use it" principle. +- **Priority:** P0 · **Effort:** S + +### 0.3 Document & test exception-safety guarantees + +- **What:** `insert` / `assign` / `emplace` / `erase` / `reorder` mutate several parallel vectors + with no stated guarantee. `reorder` ([:520-537](include/soatable/soatable.hpp#L520-L537)) builds + fresh vectors, which is good, but the multi-column sort path mutates each `ColumnVector` in turn. +- **Why:** A throwing element constructor mid-operation can leave sparse/dense/data out of sync — + silent corruption. CLAUDE.md requires a _consistent, stated_ guarantee (basic / strong / nothrow). +- **Approach:** Decide and document the guarantee per public method (`@note Strong exception +guarantee.`). Where strong is feasible (single-column assign), build-then-swap; where only basic + is feasible, say so. Add tests with a column type whose constructor throws on the Nth element. +- **Priority:** P0 · **Effort:** M + +### 0.4 ABA generation-counter wraparound + +- **What:** `row_id::generation` is `std::uint32_t` + ([soatable.hpp:44](include/soatable/soatable.hpp#L44)), incremented on each erase of a slot. +- **Why:** After 2^32 erase/reuse cycles on one slot, a stale handle can alias a live row — the + exact ABA bug the generation counter exists to prevent. Rare, but it is a correctness hole that a + "best-in-class" library must at least document and ideally detect. +- **Approach:** Document the bound explicitly. Optionally retire a slot (stop recycling) when its + generation saturates, or widen to 64-bit generation behind the `size_type` work in 0.1. +- **Priority:** P0 · **Effort:** S + +### 0.5 Namespace alias & macro hygiene + +- **What:** A compatibility alias `sstd` and feature macro `SOATABLE_HAS_PRINT` are exposed at + global/namespace scope. +- **Why:** Short global aliases and unprefixed-ish macros risk collisions in large downstream + builds and ODR surprises if the macro is defined differently across TUs. +- **Approach:** Make `sstd` opt-in (only when `SOATABLE_ENABLE_SSTD_ALIAS` is defined). Ensure every + macro is `SOATABLE_`-prefixed, `#undef`-clean, and never affects the public ABI. Add an + include-guard / single-include test. +- **Priority:** P1 · **Effort:** S + +--- + +## Phase 1 — Testing & quality gates (P0 / P1) + +### 1.1 Close the coverage gap + +- **What:** The suite is ~159 lines / ~7 tests ([tests/soatable_tests.cpp](tests/soatable_tests.cpp)) + covering insert/get, erase/validity, required & optional select, multi-column sort, batch ops, and + `quantized_float`. +- **Why:** Large swaths of the public surface are untested: `sort_by_column_parallel`, `DeltaValue`, + `packed_bits`, `DirtyMask`, the whole `RowHandle` wrapper, `shrink_to_fit`, free-list recycling + under churn, optional-column edge cases (zero required columns, all-absent driver), empty and very + large tables, and **move-only / non-trivial / throwing** column types. +- **Approach:** Add parametrized tests per public method; add a `MoveOnly` and a `ThrowsOnNth` column + type; add a churn test (insert/erase millions of times, assert slot reuse and handle invalidation). +- **Priority:** P0 · **Effort:** L + +### 1.2 Sanitizer CI jobs + +- **What:** CI builds GCC 13 / Clang 18 / MSVC but runs no AddressSanitizer or + UndefinedBehaviorSanitizer. +- **Why:** CLAUDE.md mandates ASan/UBSan; the int/size casts in 0.1 and the parallel sort are exactly + the code that sanitizers catch. +- **Approach:** Add a Linux Clang matrix leg with `-fsanitize=address,undefined` running the full + CTest suite. Treat sanitizer findings as build failures. +- **Priority:** P0 · **Effort:** S + +### 1.3 Invariant / property tests + +- **What:** No test asserts the structural invariant that sparse ↔ dense ↔ data stay consistent. +- **Why:** This single invariant underpins every operation; a property test over randomized + insert/erase/assign/sort sequences is the highest-leverage safety net. +- **Approach:** After each randomized op, assert: every dense entry round-trips through sparse; + `size()` equals alive count; no dangling tombstones. Seed deterministically (CLAUDE.md: no random + values in unit tests — use fixed seeds). +- **Priority:** P1 · **Effort:** M + +### 1.4 Compile-fail and round-trip tests + +- **What:** Concepts/`static_assert`s (unique columns, registered type) have no negative tests; the + compression helpers have no round-trip fuzzing. +- **Why:** Compile-time contracts regress silently; lossy compression needs documented, tested error + bounds. +- **Approach:** Add a CMake-driven `try_compile` matrix asserting the bad cases fail. Add round-trip + tests sweeping the full value range for `quantized_float` / `DeltaValue` and asserting the bound. +- **Priority:** P1 · **Effort:** M + +--- + +## Phase 2 — API ergonomics & const-correctness (P1) + +### 2.1 Friendlier row / select type + +- **What:** `select<>()` yields `std::tuple...>` + ([detail::select_result_t](include/soatable/soatable.hpp#L172-L184)). Callers must `.get()` through + `reference_wrapper`. +- **Why:** Reference-semantic _rows_ (tuple protocol + structured bindings, like soagen) read far more + naturally: `for (auto row : t.select()) row.a += row.b;`. +- **Approach:** Offer a thin row proxy that models the tuple protocol so structured bindings yield real + references, while keeping the current tuple as a lower-level escape hatch. +- **Priority:** P1 · **Effort:** M + +### 2.2 Const-correctness sweep + +- **What:** Verify every accessor (`get`, `try_get`, `select`, `for_each_row`, `rows`) has a correct + `const` overload and that const tables yield `const` references end to end. +- **Why:** A data structure marketed for analytics must be safely shareable as `const`. +- **Approach:** Add `static_assert`-style compile tests over a `const SoaTable&` to lock the contract. +- **Priority:** P1 · **Effort:** S + +### 2.3 Error-handling contract + +- **What:** `get<>` throws on a missing column/row while `try_get<>` returns `nullptr`; the contract is + not centrally documented. +- **Why:** Predictable, hot-path failure handling matters for finance/aerospace users who avoid + exceptions. +- **Approach:** Document the throw-vs-nullptr split once, prominently. Consider an + `std::expected` accessor (C++23) for the no-exceptions audience tied to the freestanding + work in [FEATURES.md](FEATURES.md). +- **Priority:** P1 · **Effort:** M + +### 2.4 Naming consistency + +- **What:** The API mixes `snake_case` (`row_id`, `select`, `insert`) with `PascalCase` (`SoaTable`, + `RowHandle`, `ColumnVector`, `DeltaValue`, `DirtyMask`). +- **Why:** Inconsistent casing reads as unfinished and contradicts the project style guide. +- **Approach:** Pick one convention deliberately. A defensible choice is std-style lowercase for the + container-facing API (it _is_ a numpy-like container) — but whatever is chosen, apply it uniformly + and record the rationale. Treat renames as a breaking change per SemVer; provide deprecated aliases. +- **Priority:** P1 · **Effort:** M + +--- + +## Phase 3 — Build, packaging & docs (P1 / P2) + +### 3.1 CMake presets + +- **What:** No `CMakePresets.json`; users pass flags by hand. +- **Why:** CLAUDE.md asks for presets; they make the GCC/Clang/MSVC + sanitizer + Release/Debug matrix + reproducible and one-command. +- **Approach:** Add presets for each toolchain and the sanitizer leg; wire CI to consume them so CI and + local builds can't drift. **Priority:** P1 · **Effort:** S + +### 3.2 CHANGELOG.md and CONTRIBUTING.md + +- **What:** Both are absent. +- **Why:** CLAUDE.md mandates both; a "top-tier" OSS project needs a versioned changelog (SemVer) and a + contribution guide to attract and onboard contributors. +- **Approach:** Add `CHANGELOG.md` (Keep a Changelog format) starting at 0.1.0 and a `CONTRIBUTING.md` + covering build, style (clang-format/tidy), test, and PR expectations. **Priority:** P1 · **Effort:** S + +### 3.3 Distribution channels + +- **What:** Install/export exists, but there is no vcpkg port, Conan recipe, or single-header + amalgamation. +- **Why:** "Use it everywhere" requires friction-free adoption across ecosystems. +- **Approach:** Publish a vcpkg port and a Conan recipe; add a script that amalgamates the header for + drop-in use. **Priority:** P2 · **Effort:** M + +### 3.4 Reproducible benchmarks & API docs + +- **What:** README cites ~162µs vs ~2.035ms for 250k rows but without machine specs or a one-command + harness; there is no rendered API reference despite thorough Doxygen comments. +- **Why:** Unreproducible numbers undermine credibility; a published API site is table stakes. +- **Approach:** Add a `bench` preset that prints CPU/compiler/flags and emits a results file; publish a + Doxygen site to GitHub Pages in CI. **Priority:** P2 · **Effort:** M + +--- + +## Phase 4 — Performance hardening (P2) + +### 4.1 Broaden the benchmark suite + +- **What:** One scenario (sparse 3-column select) exists. +- **Why:** Insertion, erase-churn, sort, optional joins, and density sweeps are where regressions hide. +- **Approach:** Add benchmarks across those axes; add baselines against [soagen](https://marzer.github.io/soagen/) + and a hand-rolled SoA so claims are comparative, not absolute. **Priority:** P2 · **Effort:** M + +### 4.2 Parallel-sort strategy + +- **What:** `sort_by_column_parallel` spawns one `std::async` per column. +- **Why:** One task per column under-utilizes wide machines and adds launch overhead for small tables + or few columns. +- **Approach:** Back it with a thread pool; add a size threshold below which it runs serially; document + the threading model and measure the crossover. **Priority:** P2 · **Effort:** M + +### 4.3 Validate the `select` driver heuristic + +- **What:** `select` drives iteration off the smallest required column. +- **Why:** The heuristic is sound in theory but unverified across density distributions; a wrong driver + choice can dominate runtime. +- **Approach:** Add a benchmark that varies per-column density and confirms the chosen driver is optimal + (or refine the heuristic with measured selectivity). **Priority:** P2 · **Effort:** M + +--- + +## How to verify this roadmap + +- Each P0/P1 item should become a tracked issue with the acceptance test named above. +- "Done" for Phase 0/1 = sanitizer CI green, invariant/property tests passing, exception-safety + guarantees documented and tested. +- Re-run the benchmark suite (Phase 4) before/after any layout change to prove no regression. diff --git a/include/soatable/soatable.hpp b/include/soatable/soatable.hpp index 9180af3..4c76d3f 100644 --- a/include/soatable/soatable.hpp +++ b/include/soatable/soatable.hpp @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -392,12 +391,20 @@ class DirtyMask { /// This allows for O(1) insertion, deletion, and iteration over present values. template class ColumnVector { + public: + /// @brief Unsigned index type for dense and sparse positions. + /// + /// Widening this to `std::uint64_t` is the single change required for a 64-bit build; every + /// internal index derives from it. Kept at 32 bits to match `row_id::index`. + using size_type = std::uint32_t; + private: - static constexpr std::int32_t tombstone = -1; - /// @brief Maps row_index -> dense_index. - std::vector m_sparse; + /// @brief Sentinel stored in m_sparse for a row that has no value in this column. + static constexpr size_type npos = std::numeric_limits::max(); + /// @brief Maps row_index -> dense_index (npos when absent). + std::vector m_sparse; /// @brief Maps dense_index -> row_index. - std::vector m_dense; + std::vector m_dense; /// @brief The actual data stored densely. std::vector m_data; @@ -431,7 +438,7 @@ class ColumnVector { /// @param row_index The internal index of the row. /// @return True if a value is present, false otherwise. [[nodiscard]] bool contains(std::uint32_t row_index) const { - return row_index < m_sparse.size() && m_sparse[row_index] != tombstone; + return row_index < m_sparse.size() && m_sparse[row_index] != npos; } /// @brief Try to get a pointer to the value at the given row index. @@ -441,7 +448,7 @@ class ColumnVector { if (!contains(row_index)) { return nullptr; } - return &m_data[static_cast(m_sparse[row_index])]; + return &m_data[m_sparse[row_index]]; } /// @brief Try to get a const pointer to the value at the given row index. @@ -451,7 +458,7 @@ class ColumnVector { if (!contains(row_index)) { return nullptr; } - return &m_data[static_cast(m_sparse[row_index])]; + return &m_data[m_sparse[row_index]]; } /// @brief Get a reference to the value at the given row index. Asserts if not present. @@ -459,7 +466,7 @@ class ColumnVector { /// @return Reference to the value. T& get(std::uint32_t row_index) { assert(contains(row_index) && "Out of bounds sparse column reference."); - return m_data[static_cast(m_sparse[row_index])]; + return m_data[m_sparse[row_index]]; } /// @brief Get a const reference to the value at the given row index. Asserts if not present. @@ -467,7 +474,7 @@ class ColumnVector { /// @return Const reference to the value. const T& get(std::uint32_t row_index) const { assert(contains(row_index) && "Out of bounds sparse column reference."); - return m_data[static_cast(m_sparse[row_index])]; + return m_data[m_sparse[row_index]]; } /// @brief Emplace a value at the given row index. @@ -475,21 +482,27 @@ class ColumnVector { /// @param row_index The internal index of the row. /// @param args Arguments to pass to the constructor of T. /// @return Reference to the emplaced value. + /// @note Strong exception guarantee when inserting a new value: the dense storage is grown and + /// only committed through the sparse slot once every throwing step has succeeded. Overwriting an + /// existing value offers the guarantee of T's assignment (strong for nothrow-assignable types). template T& emplace(std::uint32_t row_index, Args&&... args) { if (row_index >= m_sparse.size()) { - m_sparse.resize(static_cast(row_index) + 1, tombstone); + m_sparse.resize(static_cast(row_index) + 1, npos); } - if (m_sparse[row_index] != tombstone) { - const std::size_t dense_index = static_cast(m_sparse[row_index]); - m_data[dense_index] = T(std::forward(args)...); + if (m_sparse[row_index] != npos) { + const size_type dense_index = m_sparse[row_index]; + m_data[dense_index] = T(std::forward(args)...); return m_data[dense_index]; } - m_sparse[row_index] = static_cast(m_dense.size()); - m_dense.push_back(row_index); + // Reserve the back-mapping first so its push_back cannot reallocate (and thus cannot throw) + // after the data element is constructed; the sparse slot is written last as the commit. + m_dense.reserve(m_dense.size() + 1); m_data.emplace_back(std::forward(args)...); + m_dense.push_back(row_index); + m_sparse[row_index] = static_cast(m_dense.size() - 1); return m_data.back(); } @@ -500,43 +513,47 @@ class ColumnVector { return; } - const std::size_t dense_index = static_cast(m_sparse[row_index]); - const std::uint32_t last_row_index = m_dense.back(); + const size_type dense_index = m_sparse[row_index]; + const size_type last_row_index = m_dense.back(); - // Swap with last + // Swap with last to keep the dense arrays contiguous. if (dense_index != m_data.size() - 1) { m_data[dense_index] = std::move(m_data.back()); m_dense[dense_index] = m_dense.back(); - m_sparse[last_row_index] = static_cast(dense_index); + m_sparse[last_row_index] = dense_index; } - m_sparse[row_index] = tombstone; + m_sparse[row_index] = npos; m_data.pop_back(); m_dense.pop_back(); } /// @brief Physically reorder the dense storage based on a new row order. /// @param row_order The new order of row indices. + /// @note Basic exception guarantee: if a contained element's move constructor throws, the + /// column is left in a valid but unspecified state. The index rebuild after the move loop is + /// no-throw, so the sparse/dense/data arrays are never left mutually inconsistent. void reorder(const std::vector& row_order) { - std::vector new_data; - std::vector new_dense; + std::vector new_data; + std::vector new_dense; new_data.reserve(m_data.size()); new_dense.reserve(m_dense.size()); - for (const std::uint32_t row_index : row_order) { + for (const size_type row_index : row_order) { if (contains(row_index)) { new_data.push_back(std::move(get(row_index))); new_dense.push_back(row_index); } } - std::fill(m_sparse.begin(), m_sparse.end(), tombstone); + std::vector new_sparse(m_sparse.size(), npos); for (std::size_t i = 0; i < new_dense.size(); ++i) { - m_sparse[new_dense[i]] = static_cast(i); + new_sparse[new_dense[i]] = static_cast(i); } - m_data = std::move(new_data); - m_dense = std::move(new_dense); + m_sparse = std::move(new_sparse); + m_data = std::move(new_data); + m_dense = std::move(new_dense); } /// @brief Number of present values in the column. @@ -588,6 +605,8 @@ class SoaTable { private: static constexpr std::uint32_t npos = std::numeric_limits::max(); + /// @brief Generation value at which a slot is retired instead of recycled (ABA wraparound bound). + static constexpr std::uint32_t max_generation = std::numeric_limits::max(); /// @brief Metadata for all row slots. std::vector m_rows; @@ -678,6 +697,9 @@ class SoaTable { /// @brief Insert a new empty row and return its ID. /// @return The row_id of the newly created row. + /// @note Strong exception guarantee. In the grow path both parallel arrays are reserved before + /// either is mutated, so a `bad_alloc` cannot leave them at mismatched sizes. + /// @throws std::overflow_error If the row_id index space is exhausted. [[nodiscard]] row_id insert() { std::uint32_t index = 0; if (m_free_head != npos) { @@ -689,6 +711,8 @@ class SoaTable { if (!can_address_rows(m_rows.size() + 1)) { throw std::overflow_error("SoaTable exhausted the row_id index space."); } + m_rows.reserve(m_rows.size() + 1); + m_free_links.reserve(m_free_links.size() + 1); index = static_cast(m_rows.size()); m_rows.push_back(RowMeta {}); m_rows.back().alive = true; @@ -712,6 +736,12 @@ class SoaTable { /// @brief Erase a row by ID. /// @param id The row_id of the row to erase. + /// @note Basic exception guarantee: column element removal relies on T's move assignment, which + /// may throw for non-nothrow-movable types, leaving the table valid but in an unspecified state. + /// @note ABA safety: a slot's generation defeats stale handles for up to 2^32 erase/reuse cycles. + /// When that counter would wrap, the slot is retired (never recycled) so a saturated stale handle + /// can never alias a freshly inserted row. Retiring leaks one slot, which is acceptable given the + /// cycle count required to reach it. void erase(row_id id) { if (!is_valid(id)) { return; @@ -728,11 +758,15 @@ class SoaTable { auto& meta = m_rows[index]; meta.alive = false; - ++meta.generation; meta.signature.reset(); - m_free_links[index] = m_free_head; - m_free_head = index; + if (meta.generation == max_generation) { + // Retire the saturated slot: leave it dead and off the free list forever. + } else { + ++meta.generation; + m_free_links[index] = m_free_head; + m_free_head = index; + } --m_alive_count; } @@ -743,6 +777,9 @@ class SoaTable { /// @param id The row_id of the row. /// @param args Arguments to pass to the constructor of T. /// @return Reference to the assigned value. + /// @note Strong exception guarantee when adding a new column value (delegates to the strong path + /// of ColumnVector::emplace); overwriting an existing value inherits T's assignment guarantee. + /// @throws std::out_of_range If the row_id is invalid (thrown before any mutation). template requires registered_column_v T& assign(row_id id, Args&&... args) { @@ -1023,6 +1060,8 @@ class SoaTable { /// criteria. /// @tparam SortCriteria The types of the comparison criteria. /// @param criteria The comparison criteria. Each should be a pair of (ColumnType, Comparator). + /// @note Basic exception guarantee: each column is reordered in turn (see ColumnVector::reorder), + /// so a throwing element move leaves the table valid but only partially reordered. template void sort_by_multi(SortCriteria&&... criteria) { // Find smallest driver if possible, otherwise use all alive rows @@ -1076,6 +1115,7 @@ class SoaTable { /// @tparam T The type of the column to sort by. /// @tparam Compare The type of the comparator. /// @param comp The comparator to use for sorting values of type T. + /// @note Basic exception guarantee (see ColumnVector::reorder). template requires registered_column_v void sort_by_column(Compare&& comp) { @@ -1104,6 +1144,8 @@ class SoaTable { /// @tparam T The type of the column to sort by. /// @tparam Compare The type of the comparator. /// @param comp The comparator to use for sorting values of type T. + /// @note Basic exception guarantee. An exception thrown while reordering a column is rethrown by + /// future::get after all tasks join, leaving the table valid but partially reordered. template requires registered_column_v void sort_by_column_parallel(Compare&& comp) { @@ -1267,5 +1309,10 @@ struct RowHandle { } // namespace soatable -/// @brief Compatibility alias for the old namespace. +#ifdef SOATABLE_ENABLE_SSTD_ALIAS +/// @brief Opt-in compatibility alias for the old namespace. +/// +/// Define `SOATABLE_ENABLE_SSTD_ALIAS` before including this header to expose `sstd`. It is opt-in +/// because a short top-level namespace alias is prone to collisions in large downstream builds. namespace sstd = soatable; +#endif diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 3582c28..2067a17 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -11,8 +11,65 @@ if(NOT GTest_FOUND) FetchContent_MakeAvailable(GTest) endif() -add_executable(soatable_tests soatable_tests.cpp) +# Strict warnings for SoaTable's own targets only; GoogleTest headers are not held to -Werror. +function(soatable_apply_strict_warnings target treat_as_errors) + if(MSVC) + target_compile_options(${target} PRIVATE /W4) + if(treat_as_errors) + target_compile_options(${target} PRIVATE /WX) + endif() + else() + target_compile_options(${target} PRIVATE -Wall -Wextra -Wconversion -Wsign-conversion) + if(treat_as_errors) + target_compile_options(${target} PRIVATE -Werror) + endif() + endif() +endfunction() + +# --- Unit tests ------------------------------------------------------------------------------- +set(SOATABLE_UNIT_TEST_SOURCES + unittests/table_test.cpp + unittests/select_test.cpp + unittests/sort_test.cpp + unittests/batch_test.cpp + unittests/row_handle_test.cpp + unittests/compression_test.cpp + unittests/exception_safety_test.cpp + unittests/invariant_test.cpp + unittests/include_hygiene_test.cpp + unittests/include_hygiene_other_tu.cpp +) + +add_executable(soatable_tests ${SOATABLE_UNIT_TEST_SOURCES}) target_link_libraries(soatable_tests PRIVATE soatable::soatable gtest_main) +target_include_directories(soatable_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/unittests) +soatable_apply_strict_warnings(soatable_tests FALSE) include(GoogleTest) gtest_discover_tests(soatable_tests) + +# --- Strict-warning compile gate (header only, treated as errors) ----------------------------- +add_executable(soatable_warning_check warning_check.cpp) +target_link_libraries(soatable_warning_check PRIVATE soatable::soatable) +soatable_apply_strict_warnings(soatable_warning_check TRUE) + +# --- Compile-fail tests: each target must fail to build (negative compile-time contracts) ------ +set(SOATABLE_COMPILE_FAIL_CASES + duplicate_columns + unregistered_column + empty_table +) + +foreach(case IN LISTS SOATABLE_COMPILE_FAIL_CASES) + add_executable(soatable_compile_fail_${case} EXCLUDE_FROM_ALL compile_fail/${case}.cpp) + target_link_libraries(soatable_compile_fail_${case} PRIVATE soatable::soatable) + set_target_properties(soatable_compile_fail_${case} PROPERTIES + EXCLUDE_FROM_DEFAULT_BUILD TRUE + ) + add_test( + NAME compile_fail_${case} + COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} + --target soatable_compile_fail_${case} --config $ + ) + set_tests_properties(compile_fail_${case} PROPERTIES WILL_FAIL TRUE) +endforeach() diff --git a/tests/compile_fail/duplicate_columns.cpp b/tests/compile_fail/duplicate_columns.cpp new file mode 100644 index 0000000..c431e43 --- /dev/null +++ b/tests/compile_fail/duplicate_columns.cpp @@ -0,0 +1,15 @@ +// Must fail to compile: a table may not register the same column type twice (is_unique). + +#include "soatable/soatable.hpp" + +namespace { +struct Age { + int value = 0; +}; +} // namespace + +int main() { + soatable::SoaTable table; + static_cast(table); + return 0; +} diff --git a/tests/compile_fail/empty_table.cpp b/tests/compile_fail/empty_table.cpp new file mode 100644 index 0000000..ab40ccd --- /dev/null +++ b/tests/compile_fail/empty_table.cpp @@ -0,0 +1,9 @@ +// Must fail to compile: a table must declare at least one column type. + +#include "soatable/soatable.hpp" + +int main() { + soatable::SoaTable<> table; + static_cast(table); + return 0; +} diff --git a/tests/compile_fail/unregistered_column.cpp b/tests/compile_fail/unregistered_column.cpp new file mode 100644 index 0000000..3378d6c --- /dev/null +++ b/tests/compile_fail/unregistered_column.cpp @@ -0,0 +1,20 @@ +// Must fail to compile: accessing a column type that is not part of the table's schema violates the +// registered_column_v constraint. + +#include "soatable/soatable.hpp" + +namespace { +struct Age { + int value = 0; +}; +struct Score { + float value = 0.0F; +}; +} // namespace + +int main() { + soatable::SoaTable table; + const auto id = table.insert(); + static_cast(table.get(id)); + return 0; +} diff --git a/tests/soatable_tests.cpp b/tests/soatable_tests.cpp deleted file mode 100644 index 072a76d..0000000 --- a/tests/soatable_tests.cpp +++ /dev/null @@ -1,159 +0,0 @@ -#include - -#include -#include -#include -#include - -#include "soatable/soatable.hpp" - -struct Name { - std::string value; -}; -struct Age { - int value; -}; -struct Score { - float value; -}; - -using TestTable = soatable::SoaTable; - -TEST(SoaTableTest, BasicInsertAndGet) { - TestTable table; - auto id = table.insert(); - EXPECT_TRUE(table.is_valid(id)); - - table.assign(id, "Alice"); - table.assign(id, 30); - - EXPECT_EQ(table.get(id).value, "Alice"); - EXPECT_EQ(table.get(id).value, 30); - EXPECT_FALSE(table.contains(id)); -} - -TEST(SoaTableTest, EraseAndValidity) { - TestTable table; - auto id = table.insert(); - table.erase(id); - EXPECT_FALSE(table.is_valid(id)); - - auto id2 = table.insert(); - EXPECT_EQ(id.index, id2.index); - EXPECT_NE(id.generation, id2.generation); -} - -TEST(SoaTableTest, SelectRequired) { - TestTable table; - auto id1 = table.insert(); - table.assign(id1, "Alice"); - table.assign(id1, 30); - - auto id2 = table.insert(); - table.assign(id2, "Bob"); - - int count = 0; - for (auto [id, name, age] : table.select()) { - EXPECT_EQ(id, id1); - EXPECT_EQ(name.get().value, "Alice"); - EXPECT_EQ(age.get().value, 30); - count++; - } - EXPECT_EQ(count, 1); -} - -TEST(SoaTableTest, SelectOptional) { - TestTable table; - auto id1 = table.insert(); - table.assign(id1, "Alice"); - table.assign(id1, 30); - - auto id2 = table.insert(); - table.assign(id2, "Bob"); - - int count = 0; - for (auto [id, name, age_opt] : table.select>()) { - if (name.get().value == "Alice") { - EXPECT_TRUE(age_opt.has_value()); - EXPECT_EQ(age_opt->get().value, 30); - } else { - EXPECT_EQ(name.get().value, "Bob"); - EXPECT_FALSE(age_opt.has_value()); - } - count++; - } - EXPECT_EQ(count, 2); -} - -TEST(SoaTableTest, MultiColumnSort) { - TestTable table; - auto id1 = table.insert(); - table.assign(id1, "Alice"); - table.assign(id1, 30); - - auto id2 = table.insert(); - table.assign(id2, "Bob"); - table.assign(id2, 20); - - auto id3 = table.insert(); - table.assign(id3, "Alice"); - table.assign(id3, 25); - - // Sort by Name asc, then Age asc - table.sort_by_multi( - std::pair> { - {}, [](const Name& a, const Name& b) { return a.value < b.value; } - }, - std::pair> { - {}, [](const Age& a, const Age& b) { return a.value < b.value; } - } - ); - - auto view = table.select(); - auto it = view.begin(); - - { - auto [id, name, age] = *it; - EXPECT_EQ(name.get().value, "Alice"); - EXPECT_EQ(age.get().value, 25); - } - ++it; - { - auto [id, name, age] = *it; - EXPECT_EQ(name.get().value, "Alice"); - EXPECT_EQ(age.get().value, 30); - } - ++it; - { - auto [id, name, age] = *it; - EXPECT_EQ(name.get().value, "Bob"); - EXPECT_EQ(age.get().value, 20); - } -} - -TEST(SoaTableTest, BatchOps) { - TestTable table; - auto ids = table.insert_batch(3); - std::vector ages = {Age {10}, Age {20}, Age {30}}; - table.assign_batch(ids, ages.begin()); - - EXPECT_EQ(table.get(ids[0]).value, 10); - EXPECT_EQ(table.get(ids[1]).value, 20); - EXPECT_EQ(table.get(ids[2]).value, 30); -} - -TEST(UtilityTest, QuantizedFloat) { - soatable::quantized_float q(0.5); - EXPECT_NEAR(q.get(), 0.5, 0.01); - - q = 1.0; - EXPECT_NEAR(q.get(), 1.0, 0.01); - - q = 0.0; - EXPECT_NEAR(q.get(), 0.0, 0.01); -} - -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/tests/unittests/batch_test.cpp b/tests/unittests/batch_test.cpp new file mode 100644 index 0000000..e334543 --- /dev/null +++ b/tests/unittests/batch_test.cpp @@ -0,0 +1,53 @@ +// Batch insertion and single-column batch assignment, including the skip-invalid-rows path. + +#include + +#include + +#include "soatable/soatable.hpp" +#include "test_types.hpp" + +using soatable_test::Age; +using soatable_test::DemoTable; + +TEST(BatchTest, InsertBatchCreatesDistinctRows) { + DemoTable table; + const auto ids = table.insert_batch(4); + + ASSERT_EQ(ids.size(), 4U); + EXPECT_EQ(table.size(), 4U); + for (const auto id : ids) { + EXPECT_TRUE(table.is_valid(id)); + } +} + +TEST(BatchTest, AssignBatchFillsColumn) { + DemoTable table; + const auto ids = table.insert_batch(3); + const std::vector ages = {Age {10}, Age {20}, Age {30}}; + table.assign_batch(ids, ages.begin()); + + EXPECT_EQ(table.get(ids[0]).value, 10); + EXPECT_EQ(table.get(ids[1]).value, 20); + EXPECT_EQ(table.get(ids[2]).value, 30); +} + +TEST(BatchTest, AssignBatchSkipsInvalidRowsButConsumesInput) { + DemoTable table; + auto ids = table.insert_batch(3); + table.erase(ids[1]); // Middle row invalid; its input value must be consumed but not applied. + + const std::vector ages = {Age {1}, Age {2}, Age {3}}; + table.assign_batch(ids, ages.begin()); + + EXPECT_EQ(table.get(ids[0]).value, 1); + EXPECT_FALSE(table.is_valid(ids[1])); + EXPECT_EQ(table.get(ids[2]).value, 3); +} + +TEST(BatchTest, InsertBatchOfZeroIsNoOp) { + DemoTable table; + const auto ids = table.insert_batch(0); + EXPECT_TRUE(ids.empty()); + EXPECT_TRUE(table.empty()); +} diff --git a/tests/unittests/compression_test.cpp b/tests/unittests/compression_test.cpp new file mode 100644 index 0000000..da2e50e --- /dev/null +++ b/tests/unittests/compression_test.cpp @@ -0,0 +1,105 @@ +// Round-trip and bound coverage for the standalone helpers: quantized_float, packed_bits, +// DeltaValue, and DirtyMask. Sweeps use deterministic value ranges (no randomness). + +#include + +#include +#include + +#include "soatable/soatable.hpp" + +namespace { + +// Default 16-bit quantizer over [0, 1000]; the worst-case round-trip error is one quantization step. +using Quant = soatable::quantized_float; +constexpr double quant_min = 0.0; +constexpr double quant_max = 1000.0; +constexpr double quant_step = (quant_max - quant_min) / 65535.0; + +} // namespace + +TEST(QuantizedFloatTest, RoundTripStaysWithinOneStepAcrossRange) { + for (int permille = 0; permille <= 1000000; permille += 1000) { + const double original = permille / 1000.0; + const Quant q {original}; + EXPECT_LE(std::abs(q.get() - original), quant_step) + << "original=" << original; + } +} + +TEST(QuantizedFloatTest, ClampsOutOfRangeInputs) { + const Quant below {-5.0}; + EXPECT_NEAR(below.get(), quant_min, quant_step); + + const Quant above {5000.0}; + EXPECT_NEAR(above.get(), quant_max, quant_step); +} + +TEST(QuantizedFloatTest, AssignmentAndConversionOperators) { + Quant q; + q = 250.0; + EXPECT_NEAR(static_cast(q), 250.0, quant_step); +} + +TEST(PackedBitsTest, SetGetRoundTripPerField) { + using LowNibble = soatable::packed_bits; + using HighNibble = soatable::packed_bits; + + std::uint16_t container = 0; + LowNibble::set(container, 0xA); + HighNibble::set(container, 0x5); + + EXPECT_EQ(LowNibble::get(container), 0xA); + EXPECT_EQ(HighNibble::get(container), 0x5); + + // Overwriting one field must not disturb the other. + LowNibble::set(container, 0x3); + EXPECT_EQ(LowNibble::get(container), 0x3); + EXPECT_EQ(HighNibble::get(container), 0x5); +} + +TEST(DeltaValueTest, DeltaRoundTripStaysWithinOneScaleStep) { + using Delta = soatable::DeltaValue; // Scale = 0.01. + constexpr double scale = 0.01; + + Delta tracked {100.0}; + for (double target = 100.0; target <= 101.0; target += 0.05) { + const std::int16_t delta = tracked.get_delta(target); + tracked.apply_delta(delta); + EXPECT_LE(std::abs(tracked.get() - target), scale) << "target=" << target; + tracked.set(target); // Re-baseline so accumulation does not drift the assertion. + } +} + +TEST(DeltaValueTest, ConstructionAndConversion) { + soatable::DeltaValue tracked {42.0}; + EXPECT_DOUBLE_EQ(static_cast(tracked), 42.0); +} + +namespace { +enum class Flags : std::uint32_t { + position = 1U << 0U, + velocity = 1U << 1U, + color = 1U << 2U, +}; +} // namespace + +TEST(DirtyMaskTest, MarkClearAndQueryFlags) { + soatable::DirtyMask mask; + EXPECT_FALSE(mask.is_any_dirty()); + + mask.mark_dirty(Flags::position); + mask.mark_dirty(Flags::color); + EXPECT_TRUE(mask.is_dirty(Flags::position)); + EXPECT_TRUE(mask.is_dirty(Flags::color)); + EXPECT_FALSE(mask.is_dirty(Flags::velocity)); + EXPECT_TRUE(mask.is_any_dirty()); + + mask.clear_dirty(Flags::position); + EXPECT_FALSE(mask.is_dirty(Flags::position)); + EXPECT_TRUE(mask.is_dirty(Flags::color)); + + mask.reset(); + EXPECT_FALSE(mask.is_any_dirty()); + EXPECT_EQ(mask.get_mask(), 0U); +} diff --git a/tests/unittests/exception_safety_test.cpp b/tests/unittests/exception_safety_test.cpp new file mode 100644 index 0000000..71d51af --- /dev/null +++ b/tests/unittests/exception_safety_test.cpp @@ -0,0 +1,75 @@ +// Exercises the documented exception-safety guarantees with a move-only column type and a column +// whose constructor throws on a chosen insertion. + +#include + +#include + +#include "soatable/soatable.hpp" +#include "test_types.hpp" + +using soatable_test::MoveOnly; +using soatable_test::ThrowOnNth; + +TEST(ExceptionSafetyTest, MoveOnlyColumnSupportsFullLifecycle) { + soatable::SoaTable table; + + const auto first = table.insert(); + table.assign(first, 5); + EXPECT_EQ(table.get(first).value, 5); + + const auto second = table.insert(); + table.assign(second, 7); + + // Erasing the first row swap-removes it in the column, moving the last element into its slot. + table.erase(first); + EXPECT_EQ(table.get(second).value, 7); +} + +TEST(ExceptionSafetyTest, MoveOnlyColumnSurvivesReorder) { + soatable::SoaTable table; + const auto a = table.insert(); + table.assign(a, 3); + const auto b = table.insert(); + table.assign(b, 1); + const auto c = table.insert(); + table.assign(c, 2); + + table.sort_by_column( + [](const MoveOnly& lhs, const MoveOnly& rhs) { return lhs.value < rhs.value; } + ); + + EXPECT_EQ(table.get(a).value, 3); + EXPECT_EQ(table.get(b).value, 1); + EXPECT_EQ(table.get(c).value, 2); +} + +TEST(ExceptionSafetyTest, FailedAssignLeavesColumnConsistent) { + soatable::SoaTable table; + const auto id0 = table.insert(); + const auto id1 = table.insert(); + const auto id2 = table.insert(); + + ThrowOnNth::construct_budget = 2; // Third construction throws. + table.assign(id0, 10); + table.assign(id1, 20); + EXPECT_THROW(table.assign(id2, 30), std::runtime_error); + ThrowOnNth::reset_budget(); + + // Strong guarantee: the throwing assign added no phantom dense entry. + EXPECT_FALSE(table.contains(id2)); + EXPECT_EQ(table.get(id0).value, 10); + EXPECT_EQ(table.get(id1).value, 20); + + int present = 0; + for (auto [id, value] : table.select()) { + static_cast(id); + static_cast(value); + ++present; + } + EXPECT_EQ(present, 2); + + // The column remains usable after recovery. + table.assign(id2, 30); + EXPECT_EQ(table.get(id2).value, 30); +} diff --git a/tests/unittests/include_hygiene_other_tu.cpp b/tests/unittests/include_hygiene_other_tu.cpp new file mode 100644 index 0000000..c47b950 --- /dev/null +++ b/tests/unittests/include_hygiene_other_tu.cpp @@ -0,0 +1,19 @@ +// Second translation unit that also includes the header. If the header carried any non-inline +// definition, linking this TU together with include_hygiene_test.cpp would fail. + +#include + +#include "soatable/soatable.hpp" + +namespace { +struct Tag { + int value = 0; +}; +} // namespace + +std::size_t row_count_from_other_tu() { + soatable::SoaTable table; + static_cast(table.insert()); + static_cast(table.insert()); + return table.size(); +} diff --git a/tests/unittests/include_hygiene_test.cpp b/tests/unittests/include_hygiene_test.cpp new file mode 100644 index 0000000..d45bad0 --- /dev/null +++ b/tests/unittests/include_hygiene_test.cpp @@ -0,0 +1,31 @@ +// Verifies header hygiene: the header is safe to include in multiple translation units (no +// non-inline definitions causing ODR/link errors) and the opt-in sstd alias works when requested. + +#define SOATABLE_ENABLE_SSTD_ALIAS +#include "soatable/soatable.hpp" +// Include a second time to confirm the include guard makes re-inclusion a no-op. +#include "soatable/soatable.hpp" + +#include + +#include + +// Defined in include_hygiene_other_tu.cpp; both TUs include the header. +std::size_t row_count_from_other_tu(); + +namespace { +struct Tag { + int value = 0; +}; +} // namespace + +TEST(IncludeHygieneTest, SstdAliasResolvesToSoatable) { + sstd::SoaTable table; + const auto id = table.insert(); + table.assign(id, 1); + EXPECT_EQ(table.get(id).value, 1); +} + +TEST(IncludeHygieneTest, HeaderLinksAcrossTranslationUnits) { + EXPECT_EQ(row_count_from_other_tu(), 2U); +} diff --git a/tests/unittests/invariant_test.cpp b/tests/unittests/invariant_test.cpp new file mode 100644 index 0000000..4ecbd02 --- /dev/null +++ b/tests/unittests/invariant_test.cpp @@ -0,0 +1,93 @@ +// Property test: drive a deterministic (fixed-seed) sequence of insert/assign/unassign/erase/sort +// operations and, after every step, assert the table agrees with an independent reference model. +// This pins the sparse <-> dense <-> data consistency invariant that underpins every operation. + +#include + +#include +#include +#include +#include + +#include "soatable/soatable.hpp" +#include "test_types.hpp" + +using soatable_test::Age; +using soatable_test::Name; + +namespace { + +struct ModelRow { + soatable::row_id id {}; + std::optional age; +}; + +using Table = soatable::SoaTable; + +void verify_against_model(const Table& table, const std::vector& model) { + ASSERT_EQ(table.size(), model.size()); + + std::size_t with_age = 0; + for (const auto& row : model) { + ASSERT_TRUE(table.is_valid(row.id)); + ASSERT_EQ(table.contains(row.id), row.age.has_value()); + if (row.age.has_value()) { + ASSERT_EQ(table.get(row.id).value, *row.age); + ++with_age; + } + } + + std::size_t selected = 0; + for (auto [id, age] : table.select()) { + static_cast(id); + static_cast(age); + ++selected; + } + ASSERT_EQ(selected, with_age); +} + +} // namespace + +TEST(InvariantTest, RandomizedOperationSequencePreservesConsistency) { + Table table; + std::vector model; + std::mt19937 rng {12345U}; // Fixed seed: deterministic, reproducible. + + constexpr int iterations = 3000; + for (int step = 0; step < iterations; ++step) { + const int op = static_cast(rng() % 100U); + + if (op < 45 || model.empty()) { + // Insert a row, assigning Age most of the time. + const auto id = table.insert(); + ModelRow row {id, std::nullopt}; + if (rng() % 4U != 0U) { + const int value = static_cast(rng() % 1000U); + table.assign(id, value); + row.age = value; + } + model.push_back(row); + } else if (op < 65) { + // Erase a random alive row. + const std::size_t victim = rng() % model.size(); + table.erase(model[victim].id); + model.erase(model.begin() + static_cast(victim)); + } else if (op < 80) { + // Reassign Age on a random row. + const std::size_t target = rng() % model.size(); + const int value = static_cast(rng() % 1000U); + table.assign(model[target].id, value); + model[target].age = value; + } else if (op < 90) { + // Unassign Age on a random row (may already be absent). + const std::size_t target = rng() % model.size(); + table.unassign(model[target].id); + model[target].age = std::nullopt; + } else { + // Physically reorder by Age; handles and values must survive. + table.sort_by_column([](const Age& a, const Age& b) { return a.value < b.value; }); + } + + verify_against_model(table, model); + } +} diff --git a/tests/unittests/row_handle_test.cpp b/tests/unittests/row_handle_test.cpp new file mode 100644 index 0000000..9c6fd19 --- /dev/null +++ b/tests/unittests/row_handle_test.cpp @@ -0,0 +1,59 @@ +// RowHandle convenience wrapper: binding, validity, column access, and unbound-handle errors. + +#include + +#include + +#include "soatable/soatable.hpp" +#include "test_types.hpp" + +using soatable_test::Age; +using soatable_test::DemoTable; +using soatable_test::Name; + +using DemoHandle = soatable::RowHandle; + +TEST(RowHandleTest, DefaultHandleIsInvalid) { + DemoHandle handle; + EXPECT_FALSE(handle.is_valid()); + EXPECT_FALSE(static_cast(handle)); +} + +TEST(RowHandleTest, BoundHandleAssignsAndReads) { + DemoTable table; + DemoHandle handle {table.insert(), table}; + ASSERT_TRUE(handle.is_valid()); + + handle.assign("Dora"); + handle.assign(40); + + EXPECT_EQ(handle.get().value, "Dora"); + EXPECT_TRUE(handle.contains()); + EXPECT_NE(handle.try_get(), nullptr); +} + +TEST(RowHandleTest, UnassignAndEraseThroughHandle) { + DemoTable table; + DemoHandle handle {table.insert(), table}; + handle.assign(1); + handle.unassign(); + EXPECT_FALSE(handle.contains()); + + EXPECT_TRUE(handle.erase()); + EXPECT_FALSE(handle.is_valid()); + EXPECT_FALSE(handle.erase()); // Already gone. +} + +TEST(RowHandleTest, UnboundHandleThrowsOnAccess) { + DemoHandle handle; + EXPECT_THROW(static_cast(handle.get()), std::logic_error); +} + +TEST(RowHandleTest, ConstHandleReadsConstReference) { + DemoTable table; + DemoHandle handle {table.insert(), table}; + handle.assign(99); + + const DemoHandle& const_handle = handle; + EXPECT_EQ(const_handle.get().value, 99); +} diff --git a/tests/unittests/select_test.cpp b/tests/unittests/select_test.cpp new file mode 100644 index 0000000..1502b3e --- /dev/null +++ b/tests/unittests/select_test.cpp @@ -0,0 +1,126 @@ +// select<>() projections: required-only, mixed optional, zero-required (all-optional) driver, +// all-absent driver column, and the rows()/for_each_row iteration helpers. + +#include + +#include + +#include "soatable/soatable.hpp" +#include "test_types.hpp" + +using soatable_test::Age; +using soatable_test::DemoTable; +using soatable_test::Name; +using soatable_test::Score; + +TEST(SelectTest, RequiredColumnsYieldOnlyFullyPopulatedRows) { + DemoTable table; + const auto id1 = table.insert(); + table.assign(id1, "Alice"); + table.assign(id1, 30); + + const auto id2 = table.insert(); + table.assign(id2, "Bob"); // No Age, so excluded from select. + + int count = 0; + for (auto [id, name, age] : table.select()) { + EXPECT_EQ(id, id1); + EXPECT_EQ(name.get().value, "Alice"); + EXPECT_EQ(age.get().value, 30); + ++count; + } + EXPECT_EQ(count, 1); +} + +TEST(SelectTest, OptionalColumnsAreIncludedWhenAbsent) { + DemoTable table; + const auto id1 = table.insert(); + table.assign(id1, "Alice"); + table.assign(id1, 30); + + const auto id2 = table.insert(); + table.assign(id2, "Bob"); + + int seen = 0; + for (auto [id, name, age_opt] : table.select>()) { + if (name.get().value == "Alice") { + ASSERT_TRUE(age_opt.has_value()); + EXPECT_EQ(age_opt->get().value, 30); + } else { + EXPECT_EQ(name.get().value, "Bob"); + EXPECT_FALSE(age_opt.has_value()); + } + ++seen; + } + EXPECT_EQ(seen, 2); +} + +TEST(SelectTest, AllOptionalSelectIteratesEveryAliveRow) { + DemoTable table; + const auto id1 = table.insert(); + table.assign(id1, 1); + const auto id2 = table.insert(); // No columns assigned at all. + static_cast(id2); + + int rows = 0; + for (auto [id, age_opt] : table.select>()) { + static_cast(id); + static_cast(age_opt); + ++rows; + } + EXPECT_EQ(rows, 2); +} + +TEST(SelectTest, AbsentDriverColumnYieldsNothing) { + DemoTable table; + const auto id = table.insert(); + table.assign(id, "Alice"); + + int count = 0; + for (auto [row, score] : table.select()) { + static_cast(row); + static_cast(score); + ++count; + } + EXPECT_EQ(count, 0); +} + +TEST(SelectTest, SelectMutatesUnderlyingColumn) { + DemoTable table; + const auto id = table.insert(); + table.assign(id, 10); + + for (auto [row, age] : table.select()) { + static_cast(row); + age.get().value += 5; + } + EXPECT_EQ(table.get(id).value, 15); +} + +TEST(SelectTest, ConstSelectYieldsConstReferences) { + DemoTable table; + const auto id = table.insert(); + table.assign(id, 42); + + const DemoTable& const_table = table; + int sum = 0; + for (auto [row, age] : const_table.select()) { + static_cast(row); + sum += age.get().value; + } + EXPECT_EQ(sum, 42); +} + +TEST(SelectTest, RowsIterationVisitsEveryAliveRow) { + DemoTable table; + const auto id1 = table.insert(); + const auto id2 = table.insert(); + table.erase(id1); + + int visited = 0; + for (const auto id : table.rows()) { + EXPECT_EQ(id, id2); + ++visited; + } + EXPECT_EQ(visited, 1); +} diff --git a/tests/unittests/sort_test.cpp b/tests/unittests/sort_test.cpp new file mode 100644 index 0000000..adec1b2 --- /dev/null +++ b/tests/unittests/sort_test.cpp @@ -0,0 +1,113 @@ +// Sorting: single-column, the sort_by alias, multi-column tie-breaking, and the parallel variant. +// Each asserts that physical reordering keeps every row_id valid and every value retrievable. + +#include + +#include +#include +#include + +#include "soatable/soatable.hpp" +#include "test_types.hpp" + +using soatable_test::Age; +using soatable_test::DemoTable; +using soatable_test::Name; + +namespace { + +DemoTable make_people() { + DemoTable table; + const auto id1 = table.insert(); + table.assign(id1, "Alice"); + table.assign(id1, 30); + + const auto id2 = table.insert(); + table.assign(id2, "Bob"); + table.assign(id2, 20); + + const auto id3 = table.insert(); + table.assign(id3, "Alice"); + table.assign(id3, 25); + return table; +} + +std::vector ages_in_order(DemoTable& table) { + std::vector ages; + for (auto [id, age] : table.select()) { + static_cast(id); + ages.push_back(age.get().value); + } + return ages; +} + +} // namespace + +TEST(SortTest, SingleColumnAscending) { + DemoTable table = make_people(); + table.sort_by_column([](const Age& a, const Age& b) { return a.value < b.value; }); + EXPECT_EQ(ages_in_order(table), (std::vector {20, 25, 30})); +} + +TEST(SortTest, SortByAliasMatchesSortByColumn) { + DemoTable table = make_people(); + table.sort_by([](const Age& a, const Age& b) { return a.value > b.value; }); + EXPECT_EQ(ages_in_order(table), (std::vector {30, 25, 20})); +} + +TEST(SortTest, MultiColumnTieBreak) { + DemoTable table = make_people(); + table.sort_by_multi( + std::pair> { + {}, [](const Name& a, const Name& b) { return a.value < b.value; } + }, + std::pair> { + {}, [](const Age& a, const Age& b) { return a.value < b.value; } + } + ); + + std::vector> ordered; + for (auto [id, name, age] : table.select()) { + static_cast(id); + ordered.emplace_back(name.get().value, age.get().value); + } + + ASSERT_EQ(ordered.size(), 3U); + EXPECT_EQ(ordered[0], (std::pair {"Alice", 25})); + EXPECT_EQ(ordered[1], (std::pair {"Alice", 30})); + EXPECT_EQ(ordered[2], (std::pair {"Bob", 20})); +} + +TEST(SortTest, ParallelSortMatchesSerialResult) { + DemoTable table = make_people(); + table.sort_by_column_parallel([](const Age& a, const Age& b) { return a.value < b.value; }); + + EXPECT_EQ(ages_in_order(table), (std::vector {20, 25, 30})); + + // Every row must remain individually retrievable by its still-valid handle after reordering. + int matched = 0; + for (auto [id, name, age] : table.select()) { + EXPECT_TRUE(table.is_valid(id)); + EXPECT_EQ(table.get(id).value, age.get().value); + static_cast(name); + ++matched; + } + EXPECT_EQ(matched, 3); +} + +TEST(SortTest, ParallelSortHandlesSingleColumnTable) { + soatable::SoaTable table; + const auto id1 = table.insert(); + table.assign(id1, 3); + const auto id2 = table.insert(); + table.assign(id2, 1); + + table.sort_by_column_parallel([](const Age& a, const Age& b) { return a.value < b.value; }); + + std::vector ages; + for (auto [id, age] : table.select()) { + static_cast(id); + ages.push_back(age.get().value); + } + EXPECT_EQ(ages, (std::vector {1, 3})); +} diff --git a/tests/unittests/table_test.cpp b/tests/unittests/table_test.cpp new file mode 100644 index 0000000..aacb909 --- /dev/null +++ b/tests/unittests/table_test.cpp @@ -0,0 +1,178 @@ +// Core table lifecycle: insert/erase/assign/get/contains/unassign, handle validity, free-list +// recycling under churn, ABA generation behaviour, and empty/large tables. + +#include + +#include +#include + +#include "soatable/soatable.hpp" +#include "test_types.hpp" + +using soatable_test::Age; +using soatable_test::DemoTable; +using soatable_test::Name; +using soatable_test::Score; + +TEST(TableTest, BasicInsertAssignAndGet) { + DemoTable table; + const auto id = table.insert(); + EXPECT_TRUE(table.is_valid(id)); + + table.assign(id, "Alice"); + table.assign(id, 30); + + EXPECT_EQ(table.get(id).value, "Alice"); + EXPECT_EQ(table.get(id).value, 30); + EXPECT_FALSE(table.contains(id)); + EXPECT_EQ(table.size(), 1U); +} + +TEST(TableTest, TryGetReturnsNullForAbsentColumn) { + DemoTable table; + const auto id = table.insert(); + table.assign(id, "Bob"); + + EXPECT_NE(table.try_get(id), nullptr); + EXPECT_EQ(table.try_get(id), nullptr); +} + +TEST(TableTest, GetThrowsForAbsentColumn) { + DemoTable table; + const auto id = table.insert(); + EXPECT_THROW(static_cast(table.get(id)), std::out_of_range); +} + +TEST(TableTest, AssignThrowsForInvalidRow) { + DemoTable table; + const auto id = table.insert(); + table.erase(id); + EXPECT_THROW(table.assign(id, 1), std::out_of_range); +} + +TEST(TableTest, UnassignRemovesColumnButKeepsRow) { + DemoTable table; + const auto id = table.insert(); + table.assign(id, 7); + ASSERT_TRUE(table.contains(id)); + + table.unassign(id); + EXPECT_FALSE(table.contains(id)); + EXPECT_TRUE(table.is_valid(id)); +} + +TEST(TableTest, EraseInvalidatesHandleAndBumpsGeneration) { + DemoTable table; + const auto id = table.insert(); + table.erase(id); + EXPECT_FALSE(table.is_valid(id)); + + const auto reused = table.insert(); + EXPECT_EQ(id.index, reused.index); + EXPECT_NE(id.generation, reused.generation); + EXPECT_FALSE(table.is_valid(id)); + EXPECT_TRUE(table.is_valid(reused)); +} + +TEST(TableTest, StaleHandleNeverAccessesRecycledRow) { + DemoTable table; + const auto first = table.insert(); + table.assign(first, 1); + table.erase(first); + + const auto second = table.insert(); + table.assign(second, 2); + + // The stale handle must not read the recycled slot's new value. + EXPECT_EQ(table.try_get(first), nullptr); + EXPECT_EQ(table.get(second).value, 2); +} + +TEST(TableTest, FreeListRecyclesSlotsUnderChurn) { + DemoTable table; + + // Repeated insert/erase of single rows must reuse the same slot rather than growing storage. + for (int iteration = 0; iteration < 1000; ++iteration) { + const auto id = table.insert(); + table.assign(id, iteration); + EXPECT_EQ(table.get(id).value, iteration); + table.erase(id); + EXPECT_FALSE(table.is_valid(id)); + } + + EXPECT_EQ(table.size(), 0U); + EXPECT_LE(table.row_slots(), 2U); +} + +TEST(TableTest, InterleavedChurnKeepsSizeConsistent) { + DemoTable table; + std::vector live; + + for (int iteration = 0; iteration < 500; ++iteration) { + const auto id = table.insert(); + table.assign(id, iteration); + live.push_back(id); + + // Erase every third row to exercise mid-array swap-removal in the columns. + if (iteration % 3 == 0 && !live.empty()) { + const auto victim = live.front(); + live.erase(live.begin()); + table.erase(victim); + } + } + + EXPECT_EQ(table.size(), live.size()); + for (const auto id : live) { + ASSERT_TRUE(table.is_valid(id)); + EXPECT_TRUE(table.contains(id)); + } +} + +TEST(TableTest, EmptyTableReportsEmpty) { + DemoTable table; + EXPECT_TRUE(table.empty()); + EXPECT_EQ(table.size(), 0U); + + int visited = 0; + table.for_each_row([&](soatable::row_id) { ++visited; }); + EXPECT_EQ(visited, 0); +} + +TEST(TableTest, ClearResetsRowsButPreservesGenerationMonotonicity) { + DemoTable table; + const auto id = table.insert(); + table.assign(id, 5); + table.clear(); + + EXPECT_TRUE(table.empty()); + EXPECT_FALSE(table.is_valid(id)); +} + +TEST(TableTest, ReserveAndShrinkPreserveData) { + DemoTable table; + table.reserve(256); + + const auto id = table.insert(); + table.assign(id, "Carol"); + table.shrink_to_fit(); + + EXPECT_EQ(table.get(id).value, "Carol"); +} + +TEST(TableTest, LargeTableRetainsEveryRow) { + DemoTable table; + constexpr int row_count = 50000; + + std::vector ids; + ids.reserve(row_count); + for (int i = 0; i < row_count; ++i) { + const auto id = table.insert(); + table.assign(id, i); + ids.push_back(id); + } + + ASSERT_EQ(table.size(), static_cast(row_count)); + for (int i = 0; i < row_count; ++i) { + EXPECT_EQ(table.get(ids[static_cast(i)]).value, i); + } +} diff --git a/tests/unittests/test_types.hpp b/tests/unittests/test_types.hpp new file mode 100644 index 0000000..cceedcf --- /dev/null +++ b/tests/unittests/test_types.hpp @@ -0,0 +1,73 @@ +#pragma once + +// Shared column types and fixtures used across the SoaTable unit tests. Kept header-only so each +// translation unit compiles them directly; the throwing type's budget is per-process state reset by +// the tests that use it. + +#include +#include + +#include "soatable/soatable.hpp" + +namespace soatable_test { + +// The original demo columns, reused so existing behavioural coverage carries over. +struct Name { + std::string value; +}; +struct Age { + int value = 0; +}; +struct Score { + float value = 0.0F; +}; + +using DemoTable = soatable::SoaTable; + +// A move-only column type proves emplace/remove/reorder never rely on copy construction. +struct MoveOnly { + int value = 0; + + MoveOnly() = default; + explicit MoveOnly(int initial) : value(initial) {} + + MoveOnly(const MoveOnly&) = delete; + MoveOnly& operator=(const MoveOnly&) = delete; + MoveOnly(MoveOnly&&) noexcept = default; + MoveOnly& operator=(MoveOnly&&) noexcept = default; + ~MoveOnly() = default; +}; + +// A column type whose constructor throws once a budget of successful constructions is exhausted. +// Copies and moves never throw, so a test can pin exactly which insertion fails and then assert the +// container's strong/basic exception guarantee. +struct ThrowOnNth { + // Number of further successful constructions allowed; a negative value means unlimited. + static inline int construct_budget = -1; + + int value = 0; + + ThrowOnNth() : ThrowOnNth(0) {} + explicit ThrowOnNth(int initial) : value(initial) { consume_budget(); } + + ThrowOnNth(const ThrowOnNth&) = default; + ThrowOnNth& operator=(const ThrowOnNth&) = default; + ThrowOnNth(ThrowOnNth&&) noexcept = default; + ThrowOnNth& operator=(ThrowOnNth&&) noexcept = default; + ~ThrowOnNth() = default; + + // Restore the unlimited default so one test cannot leak its budget into another. + static void reset_budget() noexcept { construct_budget = -1; } + + private: + static void consume_budget() { + if (construct_budget == 0) { + throw std::runtime_error("ThrowOnNth construction budget exhausted."); + } + if (construct_budget > 0) { + --construct_budget; + } + } +}; + +} // namespace soatable_test diff --git a/tests/warning_check.cpp b/tests/warning_check.cpp new file mode 100644 index 0000000..017df2d --- /dev/null +++ b/tests/warning_check.cpp @@ -0,0 +1,33 @@ +// Compiles the public header under the project's strict warning set (no GoogleTest headers in the +// translation unit) so that -Wconversion / -Wsign-conversion regressions surface as build failures. + +#include "soatable/soatable.hpp" + +namespace { +struct Position { + double value = 0.0; +}; +struct Velocity { + double value = 0.0; +}; +} // namespace + +int main() { + soatable::SoaTable table; + const auto id = table.insert(); + table.assign(id, Position {1.0}); + table.assign(id, Velocity {2.0}); + + double total = 0.0; + for (auto [row, position] : table.select()) { + static_cast(row); + total += position.get().value; + } + + table.sort_by_column( + [](const Position& lhs, const Position& rhs) { return lhs.value < rhs.value; } + ); + table.erase(id); + + return total > 0.0 ? 0 : 0; +} From 2bec592f1f77d076e1391e09a51ab5151af169eb Mon Sep 17 00:00:00 2001 From: bbalouki Date: Wed, 24 Jun 2026 15:21:08 +0100 Subject: [PATCH 02/58] Modularize build into cmake/ helper and version modules Move build options and tooling helpers (strict warnings target, clang-tidy, coverage, Doxygen, clang-format, AddressSanitizer) into cmake/Helpers.cmake, and pin the GoogleTest version in cmake/Versions.cmake. CMakeLists.txt now includes both. This captures a build-system refactor present in the working tree; it is independent of the library API work. Co-Authored-By: Claude Opus 4.8 --- CMakeLists.txt | 9 +- cmake/Helpers.cmake | 225 +++++++++++++++++++++++++++++++++++++++++++ cmake/Versions.cmake | 3 + 3 files changed, 231 insertions(+), 6 deletions(-) create mode 100644 cmake/Helpers.cmake create mode 100644 cmake/Versions.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 914c7d8..72257fc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,19 +1,16 @@ cmake_minimum_required(VERSION 3.20) -project(soatable VERSION 0.1.0 LANGUAGES CXX) +project(SOATABLE VERSION 0.1.0 LANGUAGES CXX) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) set(FETCHCONTENT_BASE_DIR "${CMAKE_SOURCE_DIR}/.cache/_deps") +include(${CMAKE_SOURCE_DIR}/cmake/Helpers.cmake) +include(${CMAKE_SOURCE_DIR}/cmake/Versions.cmake) include(GNUInstallDirs) include(CMakePackageConfigHelpers) -option(SOATABLE_BUILD_EXAMPLES "Build the SoaTable example program" ON) -option(SOATABLE_BUILD_BENCHMARKS "Build the SoaTable benchmark program" ON) -option(SOATABLE_BUILD_TESTS "Build the SoaTable tests" ON) - - add_library(soatable INTERFACE) add_library(soatable::soatable ALIAS soatable) diff --git a/cmake/Helpers.cmake b/cmake/Helpers.cmake new file mode 100644 index 0000000..a67373c --- /dev/null +++ b/cmake/Helpers.cmake @@ -0,0 +1,225 @@ +cmake_minimum_required(VERSION 3.23) + +set(CMAKE_CXX_EXTENSIONS OFF) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(${PROJECT_NAME}_CXX_STANDARD 23 CACHE STRING "C++ standard for the project") +set(CMAKE_CXX_STANDARD ${${PROJECT_NAME}_CXX_STANDARD}) + +option(${PROJECT_NAME}_BUILD_TESTS "Add tests" OFF) +option(${PROJECT_NAME}_BUILD_BENCHMARKS "Add benchmark analysis" OFF) +option(${PROJECT_NAME}_BUILD_EXAMPLES "Build some examples" OFF) + +option(${PROJECT_NAME}_ADD_COVERAGE_ANALYSIS "Enable coverage analisys" OFF) +option(${PROJECT_NAME}_APPLY_FORMATING "Apply formating with clang-format" OFF) +option(${PROJECT_NAME}_USE_PER_FILE_FORMATTING "For very large projects" OFF) +option(${PROJECT_NAME}_APPLY_CLANG_TIDY_GLOBALY "Apply clang tidy globaly" OFF) +option(${PROJECT_NAME}_BUILD_DOCUMENTATION "Build documenation with Doxygen" OFF) +option(${PROJECT_NAME}_ENABLE_ADDRESS_SANITIZER "Enable Address Sanitizer" OFF) + +###################################################### +function(set_cxx_std target_name standard) + message(STATUS + "Setting C++${standard} standard for: ${target_name}" + ) + target_compile_features(${target_name} PUBLIC cxx_std_${standard}) +endfunction() + +###################################################### +function(configure_visibility target_name) + if (WIN32 AND BUILD_SHARED_LIBS) + set_target_properties(${target_name} PROPERTIES + WINDOWS_EXPORT_ALL_SYMBOLS ON + ) + elseif (BUILD_SHARED_LIBS) + set_target_properties(${target_name} PROPERTIES + CXX_VISIBILITY_PRESET default + ) + endif() +endfunction() + +###################################################### +add_library(warnings INTERFACE) +add_library(warnings::strict ALIAS warnings) +if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") + target_compile_options(warnings INTERFACE + /W4 # Warning level 4 + /WX # Treat warnings as errors + /EHsc # Enable C++ exception handling (standard) + /permissive- # Enforce standard C++ conformance + /options:strict # unrecognized compiler options are an error + ) +elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU|AppleClang") + target_compile_options(warnings INTERFACE + -Wall # Enable most warning messages. + -Wpedantic # Enforces strict adherence to the C++ standard. + -Werror # Treat warnings as errors + -Wconversion # Warns about implicit type conversions that may cause loss of data. + -Wpedantic # Issue warnings needed for strict compliance to the standard. + -pedantic-errors # Like -pedantic but issue them as errors. + -Wsign-conversion # Warns about implicit conversions between signed and unsigned types. + -Wshadow # Warn when one variable shadows another (globally). + -Wshadow=local # Warn when one local variable shadows another local variable or parameter. + ) +endif() + +################################################### +function(apply_clang_tidy TARGET_NAME) + find_program(CLANG_TIDY_EXE "clang-tidy") + if(CLANG_TIDY_EXE) + message(STATUS "clang-tidy found: ${CLANG_TIDY_EXE}") + else() + return() + endif() + cmake_parse_arguments(TIDY "" "EXTRA_ARGS" "" ${ARGN}) + set(LINTING_COMMAND ${CLANG_TIDY_EXE}) + if(TIDY_EXTRA_ARGS) + list(APPEND LINTING_COMMAND ${TIDY_EXTRA_ARGS}) + endif() + set_target_properties(${TARGET_NAME} PROPERTIES + CXX_CLANG_TIDY "${LINTING_COMMAND}" + ) + message(STATUS "Enabled clang-tidy for target: ${TARGET_NAME}") +endfunction() + +################################################### +function(apply_clang_tidy_globaly) + find_program(CLANG_TIDY_EXE "clang-tidy") + + if(CLANG_TIDY_EXE) + message(STATUS "clang-tidy found: ${CLANG_TIDY_EXE}") + set(CMAKE_CXX_CLANG_TIDY ${CLANG_TIDY_EXE}) + else() + return() + endif() +endfunction() + +#################################################### +function(add_coverage_analysis path) + if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") + message(STATUS "Code coverage enabled") + if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug") + message(WARNING "It's recommended to use Debug build type for code coverage") + endif() + if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + add_compile_options( + -O0 -g + -fcoverage-mapping + -fcoverage-mcdc + -ftest-coverage + --coverage + ) + else() + add_compile_options(-O0 -ggdb --coverage) + endif() + add_link_options(--coverage) + endif() + + find_program(GCOVR_EXE gcovr) + if(GCOVR_EXE) + add_custom_target( + coverage + COMMAND ${GCOVR_EXE} + -r "${src_path}" + --filter "${src_path}/src/" + --html "coverage.html" + COMMENT "Generating code coverage report..." + ) + message(STATUS "Added 'coverage' target") + endif() +endfunction() + +#################################################### +function(build_documenation) + find_package(Doxygen) + if (Doxygen_FOUND) + message( + STATUS + "Found Doxygen Version " "${DOXYGEN_VERSION} " + "at ${DOXYGEN_EXECUTABLE}" + ) + configure_file( + "${PROJECT_SOURCE_DIR}/docs/Doxyfile.in" + "${CMAKE_CURRENT_BINARY_DIR}/Doxyfile" + @ONLY + ) + doxygen_add_docs( + docs + CONFIG_FILE "${CMAKE_CURRENT_BINARY_DIR}/Doxyfile" + COMMENT "Generating docs with Doxygen ..." + ) + message(STATUS "Add 'docs' target") + message(STATUS "To build the documenation, run: cmake --build . --target docs") + endif() +endfunction() + +##################################################### +function(apply_formating USE_PER_FILE_LOGIC PATH) + find_program(CLANG_FORMAT_EXE clang-format) + + if (NOT CLANG_FORMAT_EXE) + return() + endif() + + file(GLOB_RECURSE SOURCES "${PATH}/*.cpp") + file(GLOB_RECURSE HEADERS "${PATH}/*.h" "${PATH}/*.hpp") + list(APPEND ALL_FILES ${SOURCES} ${HEADERS}) + + if(NOT ALL_FILES) + add_custom_target(format COMMENT "No source files found to format.") + return() + endif() + + if (USE_PER_FILE_LOGIC) + message(STATUS "Add 'format' target (per-file mode for large projects)") + add_custom_target( + format + COMMENT "Formatting files one-by-one with clang-format ..." + ) + foreach(file ${ALL_FILES}) + add_custom_command( + TARGET format + PRE_BUILD + COMMAND ${CLANG_FORMAT_EXE} -i "${file}" + COMMENT "Formatting ${file}" + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + ) + endforeach() + else() + message(STATUS "Add 'format' target (single command mode)") + add_custom_target( + format + COMMENT "Formatting all files at once with clang-format ..." + COMMAND ${CLANG_FORMAT_EXE} -i ${ALL_FILES} + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + ) + endif() + message(STATUS "To format your code, run: cmake --build . --target format") + +endfunction() + +##################################################### +function(anable_address_sanitizer) + set(SANITIZER_SUPPORTED OFF) + if (MINGW) + if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(SANITIZER_SUPPORTED OFF) + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + set(SANITIZER_SUPPORTED ON) + endif() + elseif (MSVC OR CMAKE_CXX_COMPILER_ID MATCHES "Clang|GNU") + set(SANITIZER_SUPPORTED ON) + endif() + + if (SANITIZER_SUPPORTED) + message(STATUS "Enabling AddressSanitizer") + if(MSVC AND CMAKE_BUILD_TYPE STREQUAL "Debug") + add_compile_options(/fsanitize=address /Zi) + add_link_options(/INCREMENTAL:NO) + else() + add_compile_options(-fsanitize=address -fno-omit-frame-pointer -g) + add_link_options(-fsanitize=address) + endif() + else() + message(WARNING "Sanitizers are NOT supported with ${CMAKE_CXX_COMPILER_ID} on MinGW. Skipping.") + endif() +endfunction() diff --git a/cmake/Versions.cmake b/cmake/Versions.cmake new file mode 100644 index 0000000..aa73787 --- /dev/null +++ b/cmake/Versions.cmake @@ -0,0 +1,3 @@ +cmake_minimum_required(VERSION 3.20) + +set(GTEST_VERSION 1.17.0 CACHE STRING "") From e6ffe24ab10c730c982c9fbfe6ff57bd3aec2342 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Wed, 24 Jun 2026 15:21:28 +0100 Subject: [PATCH 03/58] API ergonomics, const-correctness, and snake_case migration (Phase 2) IMPROVEMENTS Phase 2: - 2.1 Add row_view, a reference-semantic row proxy modelling the tuple protocol, plus view<>()/const view<>() so structured bindings yield real references (no reference_wrapper .get()); select<>() stays as the lower-level escape hatch. - 2.2 Const-correctness: compile-time static_assert tests proving get/try_get/ get_expected/select yield const references through a const table. - 2.3 Document the throw vs nullptr vs expected accessor contract; add non-throwing get_expected() returning std::expected with an access_error enum (mutable and const overloads). - 2.4 Rename public types to std-style lowercase (soa_table, row_handle, column_vector, delta_value, dirty_mask) with [[deprecated]] PascalCase aliases for one migration window. Update tests, example, benchmark, README. New tests: view, expected, const-correctness, deprecated-alias (60 total). Verified: 60/60 ctest green; header clean under clang -Wconversion -Wsign-conversion -Werror and MSVC /W4 /WX; view/get_expected paths ASan+UBSan clean under churn. Co-Authored-By: Claude Opus 4.8 --- README.md | 24 +- benchmarks/soatable_benchmark.cpp | 2 +- example/soatable_example.cpp | 4 +- include/soatable/soatable.hpp | 316 ++++++++++++++++--- tests/CMakeLists.txt | 4 + tests/compile_fail/duplicate_columns.cpp | 2 +- tests/compile_fail/empty_table.cpp | 2 +- tests/compile_fail/unregistered_column.cpp | 2 +- tests/unittests/compression_test.cpp | 8 +- tests/unittests/const_correctness_test.cpp | 57 ++++ tests/unittests/deprecated_alias_test.cpp | 45 +++ tests/unittests/exception_safety_test.cpp | 6 +- tests/unittests/expected_test.cpp | 53 ++++ tests/unittests/include_hygiene_other_tu.cpp | 2 +- tests/unittests/include_hygiene_test.cpp | 2 +- tests/unittests/invariant_test.cpp | 2 +- tests/unittests/row_handle_test.cpp | 4 +- tests/unittests/sort_test.cpp | 2 +- tests/unittests/test_types.hpp | 4 +- tests/unittests/view_test.cpp | 69 ++++ tests/warning_check.cpp | 10 +- 21 files changed, 535 insertions(+), 85 deletions(-) create mode 100644 tests/unittests/const_correctness_test.cpp create mode 100644 tests/unittests/deprecated_alias_test.cpp create mode 100644 tests/unittests/expected_test.cpp create mode 100644 tests/unittests/view_test.cpp diff --git a/README.md b/README.md index 8904d53..83db290 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,13 @@ In our benchmarks, SoaTable's `select` queries are often **5x to 15x faster** th - **Batch Operations:** `insert_batch` and `assign_batch` for high-throughput data loading. - **Parallel Sorting:** Physically reorder columns in parallel based on a comparison criteria. - **Multi-Column Sorting:** Sort by primary, secondary, etc., keys. -- **Data Compression Utilities:** Includes `quantized_float`, `packed_bits`, and `DeltaValue` for compact storage. +- **Data Compression Utilities:** Includes `quantized_float`, `packed_bits`, and `delta_value` for compact storage. +- **Reference-Semantic Row Views:** `view()` yields `row_view` proxies so structured bindings give real references (`for (auto [id, a, b] : table.view()) a.x += b.y;`) without `.get()`. +- **Non-Throwing Accessors:** `get_expected()` returns `std::expected, access_error>` for hot paths and no-exceptions builds, alongside throwing `get()` and nullable `try_get()`. + +### Naming + +The public API uses std-style lowercase names (`soa_table`, `row_handle`, `column_vector`, `delta_value`, `dirty_mask`). The previous PascalCase spellings (`SoaTable`, `RowHandle`, ...) remain as `[[deprecated]]` aliases for one migration window and will be removed in a future major release. ## Quick Start @@ -75,19 +81,19 @@ struct Name { std::string value; }; struct Poisoned {}; // Component with no data int main() { - soatable::SoaTable table; + soatable::soa_table table; // Insertion auto id = table.insert(); table.assign(id, "Warrior"); table.assign(id, 100.0f); - // Efficient Selection - for (auto [row, health, name] : table.select()) { - std::println("Row {}: {} has {} health", row.index, name.get().value, health.get().value); + // Reference-semantic views: structured bindings yield real references (no .get()). + for (auto [row, health, name] : table.view()) { + std::println("Row {}: {} has {} health", row.index, name.value, health.value); } - // Optional columns + // Optional columns still use select<>(), which returns reference_wrapper tuples. for (auto [row, name, poison] : table.select>()) { if (poison) { std::println("{} is poisoned!", name.get().value); @@ -131,7 +137,7 @@ struct Maintenance { bool urgent; }; int main() { // Define the table with various column types - soatable::SoaTable, Driver, Maintenance> fleet; + soatable::soa_table, Driver, Maintenance> fleet; // 2. Batch Operations: High-throughput loading auto ids = fleet.insert_batch(1000); @@ -165,9 +171,9 @@ int main() { // 7. Data Compression: DeltaValue for smooth tracking auto altitude_id = ids[0]; - soatable::DeltaValue alt(100.0f); + soatable::delta_value alt(100.0f); alt.apply_delta(alt.get_delta(105.0f)); - fleet.assign>(altitude_id, alt); + fleet.assign>(altitude_id, alt); return 0; } diff --git a/benchmarks/soatable_benchmark.cpp b/benchmarks/soatable_benchmark.cpp index f70b69c..f2b0539 100644 --- a/benchmarks/soatable_benchmark.cpp +++ b/benchmarks/soatable_benchmark.cpp @@ -27,7 +27,7 @@ struct AOSRecord { static void BM_SoaTableSelect(benchmark::State& state) { const std::size_t row_count = static_cast(state.range(0)); - soatable::SoaTable table; + soatable::soa_table table; table.reserve(row_count); std::mt19937_64 rng(0x5A0A7EULL); diff --git a/example/soatable_example.cpp b/example/soatable_example.cpp index 34189a1..96c0dcc 100644 --- a/example/soatable_example.cpp +++ b/example/soatable_example.cpp @@ -47,9 +47,9 @@ struct ObjectName { struct IsActive {}; // Component with no data (tag) using Table = soatable:: - SoaTable; + soa_table; using Row = soatable:: - RowHandle; + row_handle; int main() { Table database; diff --git a/include/soatable/soatable.hpp b/include/soatable/soatable.hpp index 4c76d3f..2ae9cc0 100644 --- a/include/soatable/soatable.hpp +++ b/include/soatable/soatable.hpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -25,14 +26,14 @@ #define SOATABLE_HAS_PRINT 0 #endif -/// @brief The main namespace for the SoaTable library. +/// @brief The main namespace for the soa_table library. namespace soatable { // ========================================= // 1. Stable row identity and type helpers // ========================================= -/// @brief A stable identifier for a row in an SoaTable. +/// @brief A stable identifier for a row in an soa_table. /// /// Contains an index into the row metadata and a generation count to prevent /// the ABA problem (referencing a new row with an old handle). @@ -48,6 +49,14 @@ struct row_id { constexpr bool operator==(const row_id& other) const = default; }; +/// @brief Failure categories returned by the non-throwing std::expected accessors. +enum class access_error { + /// @brief The row_id is stale or out of range. + invalid_row, + /// @brief The row is valid but does not carry the requested column. + missing_column, +}; + /// @brief Helper concept to check if a type is present in a parameter pack. /// @tparam T The type to search for. /// @tparam Types The parameter pack to search in. @@ -87,7 +96,7 @@ struct index_of; template struct index_of> { static_assert( - sizeof...(Types) > 0, "SoaTable must contain at least one registered column type." + sizeof...(Types) > 0, "soa_table must contain at least one registered column type." ); /// @brief Internal function to find the index of type T in the pack. @@ -105,7 +114,7 @@ struct index_of> { /// @brief The compile-time index of type T in Types.... static constexpr std::size_t value = value_of(); static_assert( - value < sizeof...(Types), "The requested type is not registered inside the SoaTable." + value < sizeof...(Types), "The requested type is not registered inside the soa_table." ); }; @@ -166,7 +175,7 @@ template using optional_value_type_t = typename optional_value_type::type; /// @brief The return type of a selection query. -/// @tparam Self The SoaTable type (const or non-const). +/// @tparam Self The soa_table type (const or non-const). /// @tparam ReqColumns The requested column types. template using select_result_t = std::tuple< @@ -298,8 +307,8 @@ struct packed_bits { /// @tparam DeltaType The type used for the delta (e.g., int16_t). /// @tparam ScaleM1000 The scale factor multiplied by 1000. template -class DeltaValue { - static_assert(ScaleM1000 > 0, "DeltaValue requires a positive scale."); +class delta_value { + static_assert(ScaleM1000 > 0, "delta_value requires a positive scale."); private: T m_value {}; @@ -307,10 +316,10 @@ class DeltaValue { public: /// @brief Default constructor. Initializes the value to zero. - constexpr DeltaValue() = default; + constexpr delta_value() = default; /// @brief Construct with an initial value. /// @param initial The initial absolute value. - constexpr explicit DeltaValue(T initial) : m_value(initial) {} + constexpr explicit delta_value(T initial) : m_value(initial) {} /// @brief Set the absolute value. /// @param value The new absolute value. @@ -346,8 +355,8 @@ class DeltaValue { /// @tparam MaskType The unsigned integer type used for the mask. template requires std::is_enum_v -class DirtyMask { - static_assert(std::is_unsigned_v, "DirtyMask requires an unsigned mask type."); +class dirty_mask { + static_assert(std::is_unsigned_v, "dirty_mask requires an unsigned mask type."); private: MaskType m_mask = 0; @@ -390,7 +399,7 @@ class DirtyMask { /// (storing the actual data and the back-mapping to row index). /// This allows for O(1) insertion, deletion, and iteration over present values. template -class ColumnVector { +class column_vector { public: /// @brief Unsigned index type for dense and sparse positions. /// @@ -410,7 +419,7 @@ class ColumnVector { public: /// @brief Default constructor. - ColumnVector() = default; + column_vector() = default; /// @brief Reserve capacity for the vectors. /// @param capacity The number of elements to reserve space for. @@ -568,20 +577,96 @@ class ColumnVector { const std::vector& raw_data() const { return m_data; } }; +// =================================== +// 4. Reference-semantic row proxy +// =================================== + +/// @brief A lightweight, reference-semantic view of one row over a set of required columns. +/// +/// row_view models the tuple protocol, so structured bindings yield real references instead of the +/// `std::reference_wrapper` tuples that select() returns: +/// @code +/// for (auto [id, pos, vel] : table.view()) { +/// pos.x += vel.x; // pos and vel are real references into the columns. +/// } +/// // or, by column type: +/// for (auto row : table.view()) { +/// row.get().x += row.get().x; +/// } +/// @endcode +/// A row_view is invalidated by the same operations that invalidate column references (erase, sort). +template +class row_view { + public: + /// @brief Construct from a row identity and one pointer per viewed column. + /// @param id The identity of the row. + /// @param columns One pointer per column, in declaration order. + constexpr row_view(row_id id, Cols*... columns) noexcept : m_id(id), m_columns(columns...) {} + + /// @brief The identity of this row. + [[nodiscard]] constexpr row_id id() const noexcept { return m_id; } + + /// @brief Access a viewed column by its type. + /// @tparam T The column type, exactly as named in the view() call. + /// @return Reference to the column value. + template + [[nodiscard]] constexpr T& get() const noexcept { + return *std::get(m_columns); + } + + /// @brief Tuple-protocol accessor: index 0 is the row_id, indices 1..N are column references. + /// @tparam Index The structured-binding position. + template + [[nodiscard]] constexpr decltype(auto) get() const noexcept { + if constexpr (Index == 0) { + return m_id; + } else { + return (*std::get(m_columns)); + } + } + + private: + row_id m_id; + std::tuple m_columns; +}; + +/// @brief Maps a structured-binding index to the corresponding row_view element type. +/// @tparam Index The binding position (0 is the row_id). +/// @tparam Cols The viewed column types. +template +struct row_view_element { + /// @brief The element type at the given index. + using type = std::tuple_element_t>; +}; + +/// @brief Specialization for index 0, which yields the row identity. +/// @tparam Cols The viewed column types. +template +struct row_view_element<0, Cols...> { + /// @brief The element type at index 0. + using type = row_id; +}; + // =========================== -// 4. Relational column table +// 5. Relational column table // =========================== /// @brief A table where data is stored in sparse columns. /// +/// @par Error-handling contract +/// Accessors come in three flavours so callers can pick their failure mode: +/// - get() throws std::out_of_range when the column is absent on the row. +/// - try_get() returns nullptr when the column is absent or the row is invalid. +/// - get_expected() returns std::expected and never throws (for the no-exceptions audience). +/// /// @tparam Columns The types of the columns in the table. Each type must be unique. template -class SoaTable { +class soa_table { public: - static_assert(sizeof...(Columns) > 0, "SoaTable must contain at least one column type."); + static_assert(sizeof...(Columns) > 0, "soa_table must contain at least one column type."); static_assert( is_unique::value, - "All registered columns in an SoaTable must have unique types." + "All registered columns in an soa_table must have unique types." ); /// @brief Total number of registered columns. @@ -590,7 +675,7 @@ class SoaTable { using signature_type = std::bitset; /// @brief Metadata for a single row slot. - struct RowMeta { + struct row_meta { /// @brief Generation count for handle stability. std::uint32_t generation = 0; /// @brief Bitmask of present columns. @@ -609,7 +694,7 @@ class SoaTable { static constexpr std::uint32_t max_generation = std::numeric_limits::max(); /// @brief Metadata for all row slots. - std::vector m_rows; + std::vector m_rows; /// @brief Linked list of free slots. std::vector m_free_links; /// @brief Index of the first free slot. @@ -617,7 +702,7 @@ class SoaTable { /// @brief Number of active rows. std::size_t m_alive_count = 0; /// @brief Storage for each column. - std::tuple...> m_columns; + std::tuple...> m_columns; /// @brief Internal check for index overflow. [[nodiscard]] static constexpr bool can_address_rows(std::size_t count) { @@ -644,7 +729,7 @@ class SoaTable { const row_id id {row_index, self->m_rows[row_index].generation}; auto get_col = [&](std::uint32_t idx) { using ActualT = detail::optional_value_type_t; - auto* ptr = std::get>(self->m_columns).try_get(idx); + auto* ptr = std::get>(self->m_columns).try_get(idx); if constexpr (detail::is_optional_v) { using ResT = std::optional, const ActualT, ActualT>>>; @@ -662,7 +747,7 @@ class SoaTable { public: /// @brief Default constructor. - SoaTable() = default; + soa_table() = default; /// @brief Number of active rows in the table. [[nodiscard]] std::size_t size() const noexcept { return m_alive_count; } @@ -709,12 +794,12 @@ class SoaTable { m_rows[index].signature.reset(); } else { if (!can_address_rows(m_rows.size() + 1)) { - throw std::overflow_error("SoaTable exhausted the row_id index space."); + throw std::overflow_error("soa_table exhausted the row_id index space."); } m_rows.reserve(m_rows.size() + 1); m_free_links.reserve(m_free_links.size() + 1); index = static_cast(m_rows.size()); - m_rows.push_back(RowMeta {}); + m_rows.push_back(row_meta {}); m_rows.back().alive = true; m_free_links.push_back(npos); } @@ -749,7 +834,7 @@ class SoaTable { const std::uint32_t index = id.index; - auto clear_row_from_pool = [index](ColumnVector& pool) { + auto clear_row_from_pool = [index](column_vector& pool) { if (pool.contains(index)) { pool.remove(index); } @@ -778,7 +863,7 @@ class SoaTable { /// @param args Arguments to pass to the constructor of T. /// @return Reference to the assigned value. /// @note Strong exception guarantee when adding a new column value (delegates to the strong path - /// of ColumnVector::emplace); overwriting an existing value inherits T's assignment guarantee. + /// of column_vector::emplace); overwriting an existing value inherits T's assignment guarantee. /// @throws std::out_of_range If the row_id is invalid (thrown before any mutation). template requires registered_column_v @@ -787,7 +872,7 @@ class SoaTable { throw std::out_of_range("assign() called with an invalid row_id."); } - auto& pool = std::get>(m_columns); + auto& pool = std::get>(m_columns); T& value = pool.emplace(id.index, std::forward(args)...); m_rows[id.index].signature.set(index_of>::value); return value; @@ -803,7 +888,7 @@ class SoaTable { return; } - auto& pool = std::get>(m_columns); + auto& pool = std::get>(m_columns); pool.remove(id.index); m_rows[id.index].signature.reset(index_of>::value); } @@ -831,7 +916,7 @@ class SoaTable { if (!contains(id)) { return nullptr; } - return std::get>(m_columns).try_get(id.index); + return std::get>(m_columns).try_get(id.index); } /// @brief Try to get a const pointer to the value of a column for the given row. @@ -844,7 +929,7 @@ class SoaTable { if (!contains(id)) { return nullptr; } - return std::get>(m_columns).try_get(id.index); + return std::get>(m_columns).try_get(id.index); } /// @brief Get a reference to the value of a column for the given row. Throws if not present. @@ -878,6 +963,44 @@ class SoaTable { return *value; } + /// @brief Non-throwing accessor returning a column reference or an access_error. + /// @tparam T The type of the column. + /// @param id The row_id of the row. + /// @return The column reference on success, or an access_error describing the failure. + /// @note Never throws. Intended for hot paths and no-exceptions builds. + template + requires registered_column_v + [[nodiscard]] std::expected, access_error> get_expected(row_id id) { + if (!is_valid(id)) { + return std::unexpected(access_error::invalid_row); + } + T* value = std::get>(m_columns).try_get(id.index); + if (value == nullptr) { + return std::unexpected(access_error::missing_column); + } + return std::reference_wrapper(*value); + } + + /// @brief Non-throwing const accessor returning a const column reference or an access_error. + /// @tparam T The type of the column. + /// @param id The row_id of the row. + /// @return The const column reference on success, or an access_error describing the failure. + /// @note Never throws. + template + requires registered_column_v + [[nodiscard]] std::expected, access_error> get_expected( + row_id id + ) const { + if (!is_valid(id)) { + return std::unexpected(access_error::invalid_row); + } + const T* value = std::get>(m_columns).try_get(id.index); + if (value == nullptr) { + return std::unexpected(access_error::missing_column); + } + return std::reference_wrapper(*value); + } + /// @brief Iterate over all alive row IDs. [[nodiscard]] auto rows() { return rows_impl(this); } /// @brief Iterate over all alive row IDs (const). @@ -913,7 +1036,7 @@ class SoaTable { ); static_assert( (registered_column_v> && ...), - "One or more requested columns are not registered inside the SoaTable schema." + "One or more requested columns are not registered inside the soa_table schema." ); signature_type required_mask; @@ -932,7 +1055,7 @@ class SoaTable { return m_rows[row_index].alive; }) | std::views::transform([this](std::size_t row_index) { - return get_transform_func(this)( + return get_transform_func(this)( static_cast(row_index) ); }); @@ -943,7 +1066,7 @@ class SoaTable { ( [&] { if constexpr (!detail::is_optional_v) { - auto& pool = std::get>(m_columns); + auto& pool = std::get>(m_columns); if (pool.size() < min_size) { min_size = pool.size(); driver_dense = &pool.dense_rows(); @@ -960,7 +1083,7 @@ class SoaTable { }; return *driver_dense | std::views::filter(std::move(filter_func)) | - std::views::transform(get_transform_func(this)); + std::views::transform(get_transform_func(this)); } } @@ -974,7 +1097,7 @@ class SoaTable { ); static_assert( (registered_column_v> && ...), - "One or more requested columns are not registered inside the SoaTable schema." + "One or more requested columns are not registered inside the soa_table schema." ); signature_type required_mask; @@ -993,7 +1116,7 @@ class SoaTable { return m_rows[row_index].alive; }) | std::views::transform([this](std::size_t row_index) { - return get_transform_func(this)( + return get_transform_func(this)( static_cast(row_index) ); }); @@ -1004,7 +1127,7 @@ class SoaTable { ( [&] { if constexpr (!detail::is_optional_v) { - auto& pool = std::get>(m_columns); + auto& pool = std::get>(m_columns); if (pool.size() < min_size) { min_size = pool.size(); driver_dense = &pool.dense_rows(); @@ -1021,10 +1144,51 @@ class SoaTable { }; return *driver_dense | std::views::filter(std::move(filter_func)) | - std::views::transform(get_transform_func(this)); + std::views::transform(get_transform_func(this)); } } + /// @brief Select rows that have all specified columns and view them as reference-semantic rows. + /// @tparam ReqColumns The requested column types (optional columns are not supported here). + /// @return A range of row_view proxies supporting structured bindings to real references. + /// @note Built on select(); a row_view is invalidated by the same operations (erase, sort). + template + [[nodiscard]] auto view() { + static_assert(sizeof...(ReqColumns) > 0, "Must provide at least one column type to view."); + static_assert( + (!detail::is_optional_v && ...), + "view() supports required columns only; use select() for optional columns." + ); + return select() | std::views::transform([](auto projected) { + return std::apply( + [](row_id id, auto&... refs) { + return row_view(id, std::addressof(refs.get())...); + }, + projected + ); + }); + } + + /// @brief Const overload of view(); yields row_view proxies over const column references. + /// @tparam ReqColumns The requested column types (optional columns are not supported here). + /// @return A range of row_view proxies over const references. + template + [[nodiscard]] auto view() const { + static_assert(sizeof...(ReqColumns) > 0, "Must provide at least one column type to view."); + static_assert( + (!detail::is_optional_v && ...), + "view() supports required columns only; use select() for optional columns." + ); + return select() | std::views::transform([](auto projected) { + return std::apply( + [](row_id id, auto&... refs) { + return row_view(id, std::addressof(refs.get())...); + }, + projected + ); + }); + } + /// @brief Fast batch insertion of multiple rows. /// @param count The number of rows to insert. /// @return A vector of newly created row_ids. @@ -1045,7 +1209,7 @@ class SoaTable { template requires registered_column_v void assign_batch(const std::vector& ids, InputIt first) { - auto& pool = std::get>(m_columns); + auto& pool = std::get>(m_columns); auto it = first; for (auto id : ids) { if (is_valid(id)) { @@ -1060,7 +1224,7 @@ class SoaTable { /// criteria. /// @tparam SortCriteria The types of the comparison criteria. /// @param criteria The comparison criteria. Each should be a pair of (ColumnType, Comparator). - /// @note Basic exception guarantee: each column is reordered in turn (see ColumnVector::reorder), + /// @note Basic exception guarantee: each column is reordered in turn (see column_vector::reorder), /// so a throwing element move leaves the table valid but only partially reordered. template void sort_by_multi(SortCriteria&&... criteria) { @@ -1079,7 +1243,7 @@ class SoaTable { bool decided = (([&] { using T = typename std::decay_t::first_type; - auto& pool = std::get>(m_columns); + auto& pool = std::get>(m_columns); bool has_a = pool.contains(a); bool has_b = pool.contains(b); if (has_a && has_b) { @@ -1115,11 +1279,11 @@ class SoaTable { /// @tparam T The type of the column to sort by. /// @tparam Compare The type of the comparator. /// @param comp The comparator to use for sorting values of type T. - /// @note Basic exception guarantee (see ColumnVector::reorder). + /// @note Basic exception guarantee (see column_vector::reorder). template requires registered_column_v void sort_by_column(Compare&& comp) { - auto& pool = std::get>(m_columns); + auto& pool = std::get>(m_columns); const auto& dense = pool.dense_rows(); std::vector sorted_rows = dense; @@ -1154,7 +1318,7 @@ class SoaTable { return; } - auto& pool = std::get>(m_columns); + auto& pool = std::get>(m_columns); const auto& dense = pool.dense_rows(); std::vector sorted_rows = dense; @@ -1199,33 +1363,33 @@ class SoaTable { /// @brief A convenience wrapper around a row_id and a reference to its table. template -struct RowHandle { +struct row_handle { /// @brief The ID of the row. row_id id {}; /// @brief Pointer to the table containing the row. - SoaTable* table = nullptr; + soa_table* table = nullptr; /// @brief Default constructor. Creates an unbound and invalid handle. - constexpr RowHandle() = default; + constexpr row_handle() = default; /// @brief Construct from a row_id and table reference. /// @param row The row_id to wrap. /// @param table_ref The table that contains the row. - constexpr RowHandle(row_id row, SoaTable& table_ref) + constexpr row_handle(row_id row, soa_table& table_ref) : id(row), table(&table_ref) {} private: /// @brief Ensure the table is bound. - [[nodiscard]] SoaTable& require_table() { + [[nodiscard]] soa_table& require_table() { if (table == nullptr) { - throw std::logic_error("RowHandle is not bound to a table."); + throw std::logic_error("row_handle is not bound to a table."); } return *table; } /// @brief Ensure the table is bound (const). - [[nodiscard]] const SoaTable& require_table() const { + [[nodiscard]] const soa_table& require_table() const { if (table == nullptr) { - throw std::logic_error("RowHandle is not bound to a table."); + throw std::logic_error("row_handle is not bound to a table."); } return *table; } @@ -1307,6 +1471,34 @@ struct RowHandle { } }; +// ===================================== +// 6. Deprecated PascalCase type aliases +// ===================================== + +// The public API uses std-style lowercase names (soa_table, row_handle, column_vector, +// delta_value, dirty_mask). The previous PascalCase spellings remain as deprecated aliases for one +// migration window; they will be removed in a future major release. + +/// @deprecated Renamed to soa_table. +template +using SoaTable [[deprecated("Renamed to soa_table.")]] = soa_table; + +/// @deprecated Renamed to row_handle. +template +using RowHandle [[deprecated("Renamed to row_handle.")]] = row_handle; + +/// @deprecated Renamed to column_vector. +template +using ColumnVector [[deprecated("Renamed to column_vector.")]] = column_vector; + +/// @deprecated Renamed to delta_value. +template +using DeltaValue [[deprecated("Renamed to delta_value.")]] = delta_value; + +/// @deprecated Renamed to dirty_mask. +template +using DirtyMask [[deprecated("Renamed to dirty_mask.")]] = dirty_mask; + } // namespace soatable #ifdef SOATABLE_ENABLE_SSTD_ALIAS @@ -1316,3 +1508,23 @@ struct RowHandle { /// because a short top-level namespace alias is prone to collisions in large downstream builds. namespace sstd = soatable; #endif + +// Tuple-protocol opt-in for row_view so structured bindings yield real column references. +namespace std { + +/// @brief Number of structured-binding elements in a row_view (the row_id plus each column). +/// @tparam Cols The viewed column types. +template +struct tuple_size> + : std::integral_constant {}; + +/// @brief Type of the Index-th structured-binding element of a row_view. +/// @tparam Index The binding position (0 is the row_id). +/// @tparam Cols The viewed column types. +template +struct tuple_element> { + /// @brief The element type at the given index. + using type = typename soatable::row_view_element::type; +}; + +} // namespace std diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 2067a17..dbb031a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -36,6 +36,10 @@ set(SOATABLE_UNIT_TEST_SOURCES unittests/compression_test.cpp unittests/exception_safety_test.cpp unittests/invariant_test.cpp + unittests/view_test.cpp + unittests/expected_test.cpp + unittests/const_correctness_test.cpp + unittests/deprecated_alias_test.cpp unittests/include_hygiene_test.cpp unittests/include_hygiene_other_tu.cpp ) diff --git a/tests/compile_fail/duplicate_columns.cpp b/tests/compile_fail/duplicate_columns.cpp index c431e43..385e559 100644 --- a/tests/compile_fail/duplicate_columns.cpp +++ b/tests/compile_fail/duplicate_columns.cpp @@ -9,7 +9,7 @@ struct Age { } // namespace int main() { - soatable::SoaTable table; + soatable::soa_table table; static_cast(table); return 0; } diff --git a/tests/compile_fail/empty_table.cpp b/tests/compile_fail/empty_table.cpp index ab40ccd..b675fd7 100644 --- a/tests/compile_fail/empty_table.cpp +++ b/tests/compile_fail/empty_table.cpp @@ -3,7 +3,7 @@ #include "soatable/soatable.hpp" int main() { - soatable::SoaTable<> table; + soatable::soa_table<> table; static_cast(table); return 0; } diff --git a/tests/compile_fail/unregistered_column.cpp b/tests/compile_fail/unregistered_column.cpp index 3378d6c..7f3979b 100644 --- a/tests/compile_fail/unregistered_column.cpp +++ b/tests/compile_fail/unregistered_column.cpp @@ -13,7 +13,7 @@ struct Score { } // namespace int main() { - soatable::SoaTable table; + soatable::soa_table table; const auto id = table.insert(); static_cast(table.get(id)); return 0; diff --git a/tests/unittests/compression_test.cpp b/tests/unittests/compression_test.cpp index da2e50e..6da0b27 100644 --- a/tests/unittests/compression_test.cpp +++ b/tests/unittests/compression_test.cpp @@ -1,5 +1,5 @@ // Round-trip and bound coverage for the standalone helpers: quantized_float, packed_bits, -// DeltaValue, and DirtyMask. Sweeps use deterministic value ranges (no randomness). +// delta_value, and dirty_mask. Sweeps use deterministic value ranges (no randomness). #include @@ -59,7 +59,7 @@ TEST(PackedBitsTest, SetGetRoundTripPerField) { } TEST(DeltaValueTest, DeltaRoundTripStaysWithinOneScaleStep) { - using Delta = soatable::DeltaValue; // Scale = 0.01. + using Delta = soatable::delta_value; // Scale = 0.01. constexpr double scale = 0.01; Delta tracked {100.0}; @@ -72,7 +72,7 @@ TEST(DeltaValueTest, DeltaRoundTripStaysWithinOneScaleStep) { } TEST(DeltaValueTest, ConstructionAndConversion) { - soatable::DeltaValue tracked {42.0}; + soatable::delta_value tracked {42.0}; EXPECT_DOUBLE_EQ(static_cast(tracked), 42.0); } @@ -85,7 +85,7 @@ enum class Flags : std::uint32_t { } // namespace TEST(DirtyMaskTest, MarkClearAndQueryFlags) { - soatable::DirtyMask mask; + soatable::dirty_mask mask; EXPECT_FALSE(mask.is_any_dirty()); mask.mark_dirty(Flags::position); diff --git a/tests/unittests/const_correctness_test.cpp b/tests/unittests/const_correctness_test.cpp new file mode 100644 index 0000000..79a8add --- /dev/null +++ b/tests/unittests/const_correctness_test.cpp @@ -0,0 +1,57 @@ +// Const-correctness contract (Phase 2.2): compile-time assertions that const accessors yield const +// references end to end, plus a runtime read through a const table. + +#include + +#include +#include +#include + +#include "soatable/soatable.hpp" +#include "test_types.hpp" + +using soatable_test::Age; +using soatable_test::DemoTable; + +namespace { + +using row_id = soatable::row_id; + +// get respects the table's const-qualification. +static_assert(std::is_same_v< + decltype(std::declval().get(std::declval())), Age&>); +static_assert(std::is_same_v< + decltype(std::declval().get(std::declval())), + const Age&>); + +// try_get likewise. +static_assert(std::is_same_v< + decltype(std::declval().try_get(std::declval())), Age*>); +static_assert(std::is_same_v< + decltype(std::declval().try_get(std::declval())), + const Age*>); + +// get_expected wraps a const reference when called on a const table. +static_assert(std::is_same_v< + decltype(std::declval().get_expected(std::declval())), + std::expected, soatable::access_error>>); + +} // namespace + +TEST(ConstCorrectnessTest, ConstTableIsFullyReadable) { + DemoTable table; + const auto id = table.insert(); + table.assign(id, 11); + + const DemoTable& const_table = table; + EXPECT_EQ(const_table.get(id).value, 11); + EXPECT_NE(const_table.try_get(id), nullptr); + EXPECT_TRUE(const_table.contains(id)); + + int via_select = 0; + for (auto [row, age] : const_table.select()) { + static_cast(row); + via_select += age.get().value; + } + EXPECT_EQ(via_select, 11); +} diff --git a/tests/unittests/deprecated_alias_test.cpp b/tests/unittests/deprecated_alias_test.cpp new file mode 100644 index 0000000..6efa732 --- /dev/null +++ b/tests/unittests/deprecated_alias_test.cpp @@ -0,0 +1,45 @@ +// Verifies the deprecated PascalCase aliases (Phase 2.4 migration window) still resolve to the +// new snake_case types. The deprecation diagnostic is intentionally silenced for this TU, since +// exercising the deprecated path is the whole point of the test. + +#if defined(_MSC_VER) +#pragma warning(push) +#pragma warning(disable : 4996) // deprecated declaration +#elif defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + +#include + +#include + +#include "soatable/soatable.hpp" + +namespace { +struct Age { + int value = 0; +}; +enum class Flags : std::uint32_t { + dirty = 1U << 0U, +}; +} // namespace + +TEST(DeprecatedAliasTest, PascalCaseNamesResolveToSnakeCase) { + static_assert(std::is_same_v, soatable::soa_table>); + static_assert(std::is_same_v, soatable::row_handle>); + static_assert(std::is_same_v, soatable::column_vector>); + static_assert(std::is_same_v, soatable::delta_value>); + static_assert(std::is_same_v, soatable::dirty_mask>); + + soatable::SoaTable table; + const auto id = table.insert(); + table.assign(id, 3); + EXPECT_EQ(table.get(id).value, 3); +} + +#if defined(_MSC_VER) +#pragma warning(pop) +#elif defined(__GNUC__) || defined(__clang__) +#pragma GCC diagnostic pop +#endif diff --git a/tests/unittests/exception_safety_test.cpp b/tests/unittests/exception_safety_test.cpp index 71d51af..cdafe88 100644 --- a/tests/unittests/exception_safety_test.cpp +++ b/tests/unittests/exception_safety_test.cpp @@ -12,7 +12,7 @@ using soatable_test::MoveOnly; using soatable_test::ThrowOnNth; TEST(ExceptionSafetyTest, MoveOnlyColumnSupportsFullLifecycle) { - soatable::SoaTable table; + soatable::soa_table table; const auto first = table.insert(); table.assign(first, 5); @@ -27,7 +27,7 @@ TEST(ExceptionSafetyTest, MoveOnlyColumnSupportsFullLifecycle) { } TEST(ExceptionSafetyTest, MoveOnlyColumnSurvivesReorder) { - soatable::SoaTable table; + soatable::soa_table table; const auto a = table.insert(); table.assign(a, 3); const auto b = table.insert(); @@ -45,7 +45,7 @@ TEST(ExceptionSafetyTest, MoveOnlyColumnSurvivesReorder) { } TEST(ExceptionSafetyTest, FailedAssignLeavesColumnConsistent) { - soatable::SoaTable table; + soatable::soa_table table; const auto id0 = table.insert(); const auto id1 = table.insert(); const auto id2 = table.insert(); diff --git a/tests/unittests/expected_test.cpp b/tests/unittests/expected_test.cpp new file mode 100644 index 0000000..d95239e --- /dev/null +++ b/tests/unittests/expected_test.cpp @@ -0,0 +1,53 @@ +// get_expected (Phase 2.3): the non-throwing accessor returns the value on success and a typed +// access_error otherwise, in both the mutable and const overloads. + +#include + +#include "soatable/soatable.hpp" +#include "test_types.hpp" + +using soatable_test::Age; +using soatable_test::DemoTable; + +TEST(ExpectedTest, ReturnsReferenceOnSuccess) { + DemoTable table; + const auto id = table.insert(); + table.assign(id, 5); + + auto result = table.get_expected(id); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result->get().value, 5); + + result->get().value = 6; // The wrapped reference is mutable. + EXPECT_EQ(table.get(id).value, 6); +} + +TEST(ExpectedTest, ReportsMissingColumn) { + DemoTable table; + const auto id = table.insert(); + + const auto result = table.get_expected(id); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error(), soatable::access_error::missing_column); +} + +TEST(ExpectedTest, ReportsInvalidRow) { + DemoTable table; + const auto id = table.insert(); + table.erase(id); + + const auto result = table.get_expected(id); + ASSERT_FALSE(result.has_value()); + EXPECT_EQ(result.error(), soatable::access_error::invalid_row); +} + +TEST(ExpectedTest, ConstAccessorReturnsConstReference) { + DemoTable table; + const auto id = table.insert(); + table.assign(id, 9); + + const DemoTable& const_table = table; + const auto result = const_table.get_expected(id); + ASSERT_TRUE(result.has_value()); + EXPECT_EQ(result->get().value, 9); +} diff --git a/tests/unittests/include_hygiene_other_tu.cpp b/tests/unittests/include_hygiene_other_tu.cpp index c47b950..56ea74b 100644 --- a/tests/unittests/include_hygiene_other_tu.cpp +++ b/tests/unittests/include_hygiene_other_tu.cpp @@ -12,7 +12,7 @@ struct Tag { } // namespace std::size_t row_count_from_other_tu() { - soatable::SoaTable table; + soatable::soa_table table; static_cast(table.insert()); static_cast(table.insert()); return table.size(); diff --git a/tests/unittests/include_hygiene_test.cpp b/tests/unittests/include_hygiene_test.cpp index d45bad0..fee49f9 100644 --- a/tests/unittests/include_hygiene_test.cpp +++ b/tests/unittests/include_hygiene_test.cpp @@ -20,7 +20,7 @@ struct Tag { } // namespace TEST(IncludeHygieneTest, SstdAliasResolvesToSoatable) { - sstd::SoaTable table; + sstd::soa_table table; const auto id = table.insert(); table.assign(id, 1); EXPECT_EQ(table.get(id).value, 1); diff --git a/tests/unittests/invariant_test.cpp b/tests/unittests/invariant_test.cpp index 4ecbd02..d9c61c1 100644 --- a/tests/unittests/invariant_test.cpp +++ b/tests/unittests/invariant_test.cpp @@ -22,7 +22,7 @@ struct ModelRow { std::optional age; }; -using Table = soatable::SoaTable; +using Table = soatable::soa_table; void verify_against_model(const Table& table, const std::vector& model) { ASSERT_EQ(table.size(), model.size()); diff --git a/tests/unittests/row_handle_test.cpp b/tests/unittests/row_handle_test.cpp index 9c6fd19..598a0e2 100644 --- a/tests/unittests/row_handle_test.cpp +++ b/tests/unittests/row_handle_test.cpp @@ -1,4 +1,4 @@ -// RowHandle convenience wrapper: binding, validity, column access, and unbound-handle errors. +// row_handle convenience wrapper: binding, validity, column access, and unbound-handle errors. #include @@ -11,7 +11,7 @@ using soatable_test::Age; using soatable_test::DemoTable; using soatable_test::Name; -using DemoHandle = soatable::RowHandle; +using DemoHandle = soatable::row_handle; TEST(RowHandleTest, DefaultHandleIsInvalid) { DemoHandle handle; diff --git a/tests/unittests/sort_test.cpp b/tests/unittests/sort_test.cpp index adec1b2..b9a2a59 100644 --- a/tests/unittests/sort_test.cpp +++ b/tests/unittests/sort_test.cpp @@ -96,7 +96,7 @@ TEST(SortTest, ParallelSortMatchesSerialResult) { } TEST(SortTest, ParallelSortHandlesSingleColumnTable) { - soatable::SoaTable table; + soatable::soa_table table; const auto id1 = table.insert(); table.assign(id1, 3); const auto id2 = table.insert(); diff --git a/tests/unittests/test_types.hpp b/tests/unittests/test_types.hpp index cceedcf..1edeaf5 100644 --- a/tests/unittests/test_types.hpp +++ b/tests/unittests/test_types.hpp @@ -1,6 +1,6 @@ #pragma once -// Shared column types and fixtures used across the SoaTable unit tests. Kept header-only so each +// Shared column types and fixtures used across the soa_table unit tests. Kept header-only so each // translation unit compiles them directly; the throwing type's budget is per-process state reset by // the tests that use it. @@ -22,7 +22,7 @@ struct Score { float value = 0.0F; }; -using DemoTable = soatable::SoaTable; +using DemoTable = soatable::soa_table; // A move-only column type proves emplace/remove/reorder never rely on copy construction. struct MoveOnly { diff --git a/tests/unittests/view_test.cpp b/tests/unittests/view_test.cpp new file mode 100644 index 0000000..5134f9b --- /dev/null +++ b/tests/unittests/view_test.cpp @@ -0,0 +1,69 @@ +// row_view proxy (Phase 2.1): structured bindings yield real references, get() accessor, the +// required-columns filter, and the const overload yielding readable values. + +#include + +#include "soatable/soatable.hpp" +#include "test_types.hpp" + +using soatable_test::Age; +using soatable_test::DemoTable; +using soatable_test::Name; + +TEST(ViewTest, StructuredBindingsYieldMutableReferences) { + DemoTable table; + const auto id = table.insert(); + table.assign(id, 10); + table.assign(id, "Eve"); + + for (auto [row, age, name] : table.view()) { + EXPECT_EQ(row, id); + age.value += 5; // Writes straight through to the column. + EXPECT_EQ(name.value, "Eve"); + } + EXPECT_EQ(table.get(id).value, 15); +} + +TEST(ViewTest, GetByTypeAccessorMutatesColumn) { + DemoTable table; + const auto id = table.insert(); + table.assign(id, 1); + + for (auto row : table.view()) { + EXPECT_EQ(row.id(), id); + row.get().value = 42; + } + EXPECT_EQ(table.get(id).value, 42); +} + +TEST(ViewTest, OnlyVisitsRowsWithEveryRequestedColumn) { + DemoTable table; + const auto full = table.insert(); + table.assign(full, 1); + table.assign(full, "Full"); + const auto partial = table.insert(); + table.assign(partial, 2); // No Name, so excluded. + + int visited = 0; + for (auto [row, age, name] : table.view()) { + static_cast(row); + static_cast(age); + static_cast(name); + ++visited; + } + EXPECT_EQ(visited, 1); +} + +TEST(ViewTest, ConstViewYieldsReadableValues) { + DemoTable table; + const auto id = table.insert(); + table.assign(id, 7); + + const DemoTable& const_table = table; + int sum = 0; + for (auto [row, age] : const_table.view()) { + static_cast(row); + sum += age.value; + } + EXPECT_EQ(sum, 7); +} diff --git a/tests/warning_check.cpp b/tests/warning_check.cpp index 017df2d..8d08a12 100644 --- a/tests/warning_check.cpp +++ b/tests/warning_check.cpp @@ -13,15 +13,19 @@ struct Velocity { } // namespace int main() { - soatable::SoaTable table; + soatable::soa_table table; const auto id = table.insert(); table.assign(id, Position {1.0}); table.assign(id, Velocity {2.0}); double total = 0.0; - for (auto [row, position] : table.select()) { + for (auto [row, position, velocity] : table.view()) { static_cast(row); - total += position.get().value; + total += position.value + velocity.value; + } + + if (const auto found = table.get_expected(id); found.has_value()) { + total += found->get().value; } table.sort_by_column( From 453b4a0145487115769ead7faf4208cc3df78d40 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Wed, 24 Jun 2026 15:42:59 +0100 Subject: [PATCH 04/58] Build, packaging, and docs (IMPROVEMENTS Phase 3) - 3.1 Add CMakePresets.json (debug/release, gcc/clang/msvc, asan-ubsan, bench); raise the CMake minimum to 3.25 and bump the project version to 0.3.0. CI now configures, builds, and tests through these presets so local and CI builds cannot drift. - 3.2 Add CHANGELOG.md (Keep a Changelog; records the 0.1.0 -> 0.2.0 -> 0.3.0 history including the breaking snake_case renames) and CONTRIBUTING.md. - 3.3 Add a Conan recipe, a vcpkg overlay port, and scripts/amalgamate.py that emits a stamped single-header dist/soatable.hpp. - 3.4 Add docs/Doxyfile.in (wired through the build_documenation helper), a GitHub Pages docs workflow, and scripts/run_benchmarks.sh that records the toolchain and emits a JSON results file. - Update README installation section (presets, Conan, vcpkg overlay, amalgamation, docs) and the CMake version badge. Verified locally: 60/60 ctest green via the msvc preset; the bench preset builds the benchmark; the docs target generates HTML; the amalgamated header compiles clean under clang -Wconversion -Wsign-conversion -Werror. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/ci.yaml | 31 ++-- .github/workflows/docs.yaml | 45 ++++++ .gitignore | 4 + CHANGELOG.md | 65 +++++++++ CMakeLists.txt | 8 +- CMakePresets.json | 135 ++++++++++++++++++ CONTRIBUTING.md | 70 +++++++++ README.md | 45 +++++- conanfile.py | 50 +++++++ docs/Doxyfile.in | 29 ++++ packaging/vcpkg/ports/soatable/portfile.cmake | 32 +++++ packaging/vcpkg/ports/soatable/vcpkg.json | 17 +++ scripts/amalgamate.py | 71 +++++++++ scripts/run_benchmarks.sh | 34 +++++ 14 files changed, 615 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/docs.yaml create mode 100644 CHANGELOG.md create mode 100644 CMakePresets.json create mode 100644 CONTRIBUTING.md create mode 100644 conanfile.py create mode 100644 docs/Doxyfile.in create mode 100644 packaging/vcpkg/ports/soatable/portfile.cmake create mode 100644 packaging/vcpkg/ports/soatable/vcpkg.json create mode 100644 scripts/amalgamate.py create mode 100644 scripts/run_benchmarks.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bcb6feb..644975c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,23 +17,26 @@ jobs: - { name: "Ubuntu GCC 13", os: ubuntu-latest, - cxx: g++-13 + cxx: g++-13, + preset: gcc } - { name: "Ubuntu Clang 18", os: ubuntu-latest, - cxx: clang++-18 + cxx: clang++-18, + preset: clang } - { name: "Ubuntu Clang 18 (ASan+UBSan)", os: ubuntu-latest, cxx: clang++-18, - sanitizers: true + preset: asan-ubsan } - { name: "Windows MSVC", os: windows-latest, - cxx: cl + cxx: cl, + preset: msvc } steps: @@ -54,27 +57,23 @@ jobs: - name: Configure CMake run: | - if [ "${{ matrix.config.os }}" = "windows-latest" ]; then - cmake -B build -DSOATABLE_BUILD_TESTS=ON -DSOATABLE_BUILD_EXAMPLES=ON - elif [ "${{ matrix.config.sanitizers }}" = "true" ]; then - # AddressSanitizer + UndefinedBehaviorSanitizer; -fno-sanitize-recover makes UBSan abort - # so any finding fails the build. Benchmarks are excluded to keep the leg focused. - cmake -B build -DSOATABLE_BUILD_TESTS=ON -DSOATABLE_BUILD_EXAMPLES=ON -DSOATABLE_BUILD_BENCHMARKS=OFF \ - -DCMAKE_CXX_COMPILER=${{ matrix.config.cxx }} \ - -DCMAKE_CXX_FLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -fno-omit-frame-pointer" + # Linux legs pin the exact compiler version on top of the generic preset; Windows uses the + # multi-config MSVC preset as-is. + if [ "${{ runner.os }}" = "Windows" ]; then + cmake --preset ${{ matrix.config.preset }} else - cmake -B build -DSOATABLE_BUILD_TESTS=ON -DSOATABLE_BUILD_EXAMPLES=ON -DCMAKE_CXX_COMPILER=${{ matrix.config.cxx }} + cmake --preset ${{ matrix.config.preset }} -DCMAKE_CXX_COMPILER=${{ matrix.config.cxx }} fi shell: bash - name: Build - run: cmake --build build --config Release + run: cmake --build --preset ${{ matrix.config.preset }} + shell: bash - name: Run Tests run: | - cd build # Treat any sanitizer finding as a hard failure. export ASAN_OPTIONS=halt_on_error=1:abort_on_error=1 export UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 - ctest -C Release --output-on-failure + ctest --preset ${{ matrix.config.preset }} shell: bash diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml new file mode 100644 index 0000000..88c1a5a --- /dev/null +++ b/.github/workflows/docs.yaml @@ -0,0 +1,45 @@ +name: Docs + +on: + push: + branches: [ main ] + +# Allow one concurrent deployment. +concurrency: + group: pages + cancel-in-progress: true + +permissions: + contents: read + pages: write + id-token: write + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - uses: actions/checkout@v4 + + - name: Install Doxygen + run: sudo apt-get update && sudo apt-get install -y doxygen + + - name: Generate API documentation + run: | + cmake -S . -B build-docs \ + -DSOATABLE_BUILD_DOCUMENTATION=ON \ + -DSOATABLE_BUILD_TESTS=OFF \ + -DSOATABLE_BUILD_EXAMPLES=OFF \ + -DSOATABLE_BUILD_BENCHMARKS=OFF + cmake --build build-docs --target docs + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: build-docs/doxygen/html + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index 01a2aa4..9d2bab5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,13 @@ .venv/ .env build/ +build-docs/ install/ .cache/ CMakeUserPresets.json *-workspace /.vscode/*.json prompts.txt +# Generated artifacts +dist/ +bench/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..896960e --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,65 @@ +# Changelog + +All notable changes to SoaTable are documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.3.0] - 2026-06-24 + +API ergonomics and a deliberate naming migration (IMPROVEMENTS Phase 2). + +### Added + +- `row_view`, a reference-semantic row proxy that models the tuple protocol, plus + `view()` (and a `const` overload) so structured bindings yield real references + without unwrapping `std::reference_wrapper`. +- `get_expected()` non-throwing accessors returning + `std::expected, access_error>`, with an `access_error` enum, for hot + paths and no-exceptions builds. +- Compile-time const-correctness tests covering `get` / `try_get` / `get_expected` / `select`. + +### Changed (breaking) + +- Public types renamed to std-style lowercase: `SoaTable` -> `soa_table`, `RowHandle` -> `row_handle`, + `ColumnVector` -> `column_vector`, `DeltaValue` -> `delta_value`, `DirtyMask` -> `dirty_mask`. + The previous PascalCase spellings remain as `[[deprecated]]` aliases for this migration window and + will be removed in a future major release. + +### Documentation + +- Documented the accessor error-handling contract (throw vs `nullptr` vs `std::expected`). + +## [0.2.0] - 2026-06-24 + +Correctness hardening and a real test suite (IMPROVEMENTS Phase 0 and Phase 1). + +### Added + +- Comprehensive unit-test suite under `tests/unittests/` (move-only and throwing column types, + free-list churn, ABA stale-handle, large tables, select edge cases, compression round-trips, a + fixed-seed invariant/property test, compile-fail contracts, and an include-hygiene test). +- AddressSanitizer + UndefinedBehaviorSanitizer CI leg and a strict-warning compile gate. + +### Changed + +- `column_vector` now uses an unsigned `size_type` index with an explicit `npos` sentinel instead of + a signed index with a `-1` tombstone, removing all signed/unsigned mixing. +- `emplace` (new-value path) and `insert` (grow path) now provide a strong exception guarantee; every + public mutator documents its guarantee. +- Saturated row slots are now retired instead of recycled, closing the ABA generation-wraparound hole. + +### Removed (breaking) + +- The `sstd` namespace alias is now opt-in behind `SOATABLE_ENABLE_SSTD_ALIAS`. +- Dropped the unused `` include from the public header. + +## [0.1.0] + +Initial release: header-only C++23 sparse Structure-of-Arrays table with generational `row_id` +handles, smallest-driver `select<>()`, batch operations, single / multi / parallel sorting, and the +`quantized_float` / `packed_bits` / `delta_value` / `dirty_mask` helpers. + +[0.3.0]: https://github.com/bbalouki/soatable/releases/tag/v0.3.0 +[0.2.0]: https://github.com/bbalouki/soatable/releases/tag/v0.2.0 +[0.1.0]: https://github.com/bbalouki/soatable/releases/tag/v0.1.0 diff --git a/CMakeLists.txt b/CMakeLists.txt index 72257fc..45fdde3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ -cmake_minimum_required(VERSION 3.20) +cmake_minimum_required(VERSION 3.25) -project(SOATABLE VERSION 0.1.0 LANGUAGES CXX) +project(SOATABLE VERSION 0.3.0 LANGUAGES CXX) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) @@ -62,3 +62,7 @@ if(SOATABLE_BUILD_TESTS AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests) enable_testing() add_subdirectory(tests) endif() + +if(SOATABLE_BUILD_DOCUMENTATION) + build_documenation() +endif() diff --git a/CMakePresets.json b/CMakePresets.json new file mode 100644 index 0000000..ea1cd1b --- /dev/null +++ b/CMakePresets.json @@ -0,0 +1,135 @@ +{ + "version": 6, + "cmakeMinimumRequired": { + "major": 3, + "minor": 25, + "patch": 0 + }, + "configurePresets": [ + { + "name": "base", + "hidden": true, + "binaryDir": "${sourceDir}/build/${presetName}", + "cacheVariables": { + "CMAKE_EXPORT_COMPILE_COMMANDS": "ON", + "SOATABLE_BUILD_TESTS": "ON", + "SOATABLE_BUILD_EXAMPLES": "ON", + "SOATABLE_BUILD_BENCHMARKS": "OFF" + } + }, + { + "name": "debug", + "inherits": "base", + "displayName": "Debug", + "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug" } + }, + { + "name": "release", + "inherits": "base", + "displayName": "Release", + "cacheVariables": { "CMAKE_BUILD_TYPE": "Release" } + }, + { + "name": "gcc", + "inherits": "release", + "displayName": "GCC Release", + "cacheVariables": { "CMAKE_CXX_COMPILER": "g++" } + }, + { + "name": "clang", + "inherits": "release", + "displayName": "Clang Release", + "cacheVariables": { "CMAKE_CXX_COMPILER": "clang++" } + }, + { + "name": "msvc", + "inherits": "base", + "displayName": "MSVC (multi-config)", + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Windows" + } + }, + { + "name": "asan-ubsan", + "inherits": "debug", + "displayName": "Clang Address + UB Sanitizers", + "cacheVariables": { + "CMAKE_CXX_COMPILER": "clang++", + "CMAKE_CXX_FLAGS": "-fsanitize=address,undefined -fno-sanitize-recover=all -fno-omit-frame-pointer" + } + }, + { + "name": "bench", + "inherits": "release", + "displayName": "Benchmarks", + "cacheVariables": { + "SOATABLE_BUILD_BENCHMARKS": "ON", + "SOATABLE_BUILD_TESTS": "OFF", + "SOATABLE_BUILD_EXAMPLES": "OFF" + } + } + ], + "buildPresets": [ + { "name": "debug", "configurePreset": "debug", "configuration": "Debug" }, + { + "name": "release", + "configurePreset": "release", + "configuration": "Release" + }, + { "name": "gcc", "configurePreset": "gcc", "configuration": "Release" }, + { "name": "clang", "configurePreset": "clang", "configuration": "Release" }, + { "name": "msvc", "configurePreset": "msvc", "configuration": "Release" }, + { + "name": "asan-ubsan", + "configurePreset": "asan-ubsan", + "configuration": "Debug" + }, + { "name": "bench", "configurePreset": "bench", "configuration": "Release" } + ], + "testPresets": [ + { + "name": "common", + "hidden": true, + "output": { "outputOnFailure": true }, + "execution": { "noTestsAction": "error", "stopOnFailure": false } + }, + { + "name": "debug", + "inherits": "common", + "configurePreset": "debug", + "configuration": "Debug" + }, + { + "name": "release", + "inherits": "common", + "configurePreset": "release", + "configuration": "Release" + }, + { + "name": "gcc", + "inherits": "common", + "configurePreset": "gcc", + "configuration": "Release" + }, + { + "name": "clang", + "inherits": "common", + "configurePreset": "clang", + "configuration": "Release" + }, + { + "name": "msvc", + "inherits": "common", + "configurePreset": "msvc", + "configuration": "Release" + }, + { + "name": "asan-ubsan", + "inherits": "common", + "configurePreset": "asan-ubsan", + "configuration": "Debug" + } + ] +} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..d684a8d --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,70 @@ +# Contributing to SoaTable + +Thanks for your interest in improving SoaTable. This guide covers how to build, test, and submit +changes. See [IMPROVEMENTS.md](IMPROVEMENTS.md) and [FEATURES.md](FEATURES.md) for the roadmap. + +## Prerequisites + +- A C++23 compiler: GCC 13+, Clang 18+, or MSVC (VS 2022). +- CMake 3.25 or newer. +- Optional tooling: `clang-format`, `clang-tidy`, `doxygen`, and a sanitizer-capable Clang. + +GoogleTest and Google Benchmark are fetched automatically via `FetchContent` when not found. + +## Building and testing with presets + +The repository ships a `CMakePresets.json`. The common workflows: + +```sh +# Release build + tests (pick the preset matching your compiler) +cmake --preset clang # or: gcc, msvc +cmake --build --preset clang +ctest --preset clang + +# Address + UndefinedBehavior sanitizers (Clang) +cmake --preset asan-ubsan +cmake --build --preset asan-ubsan +ctest --preset asan-ubsan + +# Benchmarks +cmake --preset bench +cmake --build --preset bench +``` + +The CI matrix (GCC 13, Clang 18, Clang ASan+UBSan, MSVC) consumes these same presets, so a green +local preset run should match CI. + +## Coding standards + +- Follow the rules in [CLAUDE.md](CLAUDE.md): C++23, `m_`-prefixed private members, `snake_case` + functions/variables and types, `PascalCase` reserved for nothing in the public API (the container + API is std-style lowercase), trailing return types, brace initialization, and `static_cast` over + C-style casts. No magic literals. +- The public header must stay warning-clean under `-Wall -Wextra -Wconversion -Wsign-conversion + -Werror` (Clang/GCC) and `/W4 /WX` (MSVC). The `soatable_warning_check` target enforces this. +- Format with `clang-format` (a `format` target is available when `clang-format` is installed): + + ```sh + cmake --build build --target format + ``` + +- `clang-tidy` checks (`cppcoreguidelines-*`, `modernize-*`, `readability-*`) are configured in + `.clang-tidy`. + +## Tests + +- Add tests alongside any change, under `tests/unittests/`, one file per surface area. +- Use real implementations; only mock external dependencies. +- Use deterministic inputs (fixed seeds), never nondeterministic randomness. +- Negative compile-time contracts go under `tests/compile_fail/` and are run as `WILL_FAIL` tests. +- All tests must pass under the sanitizer preset before submitting. + +## Pull requests + +1. Branch from `main`. +2. Keep the change focused; update [CHANGELOG.md](CHANGELOG.md) under the appropriate section. +3. Mark any backward-incompatible public API change as breaking and follow SemVer; provide a + deprecated alias and a migration note where feasible. +4. Ensure CI is green (all compilers + the sanitizer leg) and that new public API has Doxygen `///` + comments. +5. Open the PR against `main` with a clear description of the motivation and approach. diff --git a/README.md b/README.md index 83db290..a666998 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Build](https://github.com/bbalouki/soatable/actions/workflows/ci.yaml/badge.svg)](https://github.com/bbalouki/soatable/actions/workflows/ci.yaml) ![C++](https://img.shields.io/badge/C%2B%2B-23-blue.svg) -![CMake](https://img.shields.io/badge/CMake-3.20%2B-green.svg) +![CMake](https://img.shields.io/badge/CMake-3.25%2B-green.svg) ![License](https://img.shields.io/badge/License-MIT-yellow.svg) ![Header-only](https://img.shields.io/badge/Header--only-Yes-blue) ![Dependencies](https://img.shields.io/badge/Dependencies-None-brightgreen) @@ -181,18 +181,57 @@ int main() { ## Installation +SoaTable is header-only and requires a C++23 compiler and CMake 3.25+. + ### CMake -Add the repository as a subdirectory or install it: +Consume an installed package or add the repository as a subdirectory: ```cmake find_package(soatable CONFIG REQUIRED) target_link_libraries(your_target PRIVATE soatable::soatable) ``` +The repository ships `CMakePresets.json`. To build and test locally: + +```sh +cmake --preset clang # or: gcc, msvc +cmake --build --preset clang +ctest --preset clang +``` + +### Conan + +A header-only recipe is provided at the repository root: + +```sh +conan create . +``` + ### vcpkg -SoaTable is designed to be easily integrated into vcpkg. You can add it to your `vcpkg.json` once published or use it as an overlay. +Use the overlay port under `packaging/vcpkg/ports`: + +```sh +vcpkg install soatable --overlay-ports=packaging/vcpkg/ports +``` + +### Single-header drop-in + +SoaTable is already a single header. To generate a stamped, self-contained copy: + +```sh +python scripts/amalgamate.py # writes dist/soatable.hpp +``` + +### Documentation + +API reference is published to GitHub Pages from the `main` branch. Build it locally with: + +```sh +cmake -S . -B build-docs -DSOATABLE_BUILD_DOCUMENTATION=ON +cmake --build build-docs --target docs +``` ## Benchmarks diff --git a/conanfile.py b/conanfile.py new file mode 100644 index 0000000..ac7bd2d --- /dev/null +++ b/conanfile.py @@ -0,0 +1,50 @@ +"""Conan recipe for SoaTable, a header-only C++23 sparse Structure-of-Arrays table.""" + +import os + +from conan import ConanFile +from conan.tools.cmake import cmake_layout +from conan.tools.files import copy + + +class SoaTableConan(ConanFile): + name = "soatable" + version = "0.3.0" + license = "MIT" + url = "https://github.com/bbalouki/soatable" + homepage = "https://github.com/bbalouki/soatable" + description = "Header-only C++23 sparse Structure-of-Arrays table with generational handles." + topics = ("soa", "ecs", "columnar", "header-only", "cpp23") + + settings = "os", "arch", "compiler", "build_type" + no_copy_source = True + exports_sources = "include/*", "LICENSE" + + def package_id(self): + # Header-only: the package is identical across configurations. + self.info.clear() + + def layout(self): + cmake_layout(self) + + def package(self): + copy( + self, + "*.hpp", + src=os.path.join(self.source_folder, "include"), + dst=os.path.join(self.package_folder, "include"), + ) + copy( + self, + "LICENSE", + src=self.source_folder, + dst=os.path.join(self.package_folder, "licenses"), + ) + + def package_info(self): + self.cpp_info.bindirs = [] + self.cpp_info.libdirs = [] + self.cpp_info.set_property("cmake_file_name", "soatable") + self.cpp_info.set_property("cmake_target_name", "soatable::soatable") + # SoaTable requires C++23 from consumers. + self.cpp_info.cppstd = "23" diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in new file mode 100644 index 0000000..0f9bd65 --- /dev/null +++ b/docs/Doxyfile.in @@ -0,0 +1,29 @@ +# Doxygen configuration template. CMake's build_documenation() helper substitutes the @VARS@ via +# configure_file(@ONLY) and drives generation through the `docs` target. + +PROJECT_NAME = "SoaTable" +PROJECT_NUMBER = @PROJECT_VERSION@ +PROJECT_BRIEF = "Header-only C++23 sparse Structure-of-Arrays table" + +OUTPUT_DIRECTORY = @CMAKE_CURRENT_BINARY_DIR@/doxygen +INPUT = @CMAKE_SOURCE_DIR@/include @CMAKE_SOURCE_DIR@/README.md +USE_MDFILE_AS_MAINPAGE = @CMAKE_SOURCE_DIR@/README.md +RECURSIVE = YES + +EXTRACT_ALL = YES +EXTRACT_STATIC = YES +JAVADOC_AUTOBRIEF = YES +MARKDOWN_SUPPORT = YES +SORT_MEMBER_DOCS = NO + +GENERATE_HTML = YES +HTML_OUTPUT = html +GENERATE_LATEX = NO + +QUIET = YES +WARN_IF_UNDOCUMENTED = NO +WARNINGS = YES + +# C++23 sources; help Doxygen parse modern constructs. +BUILTIN_STL_SUPPORT = YES +CPP_CLI_SUPPORT = NO diff --git a/packaging/vcpkg/ports/soatable/portfile.cmake b/packaging/vcpkg/ports/soatable/portfile.cmake new file mode 100644 index 0000000..92c0687 --- /dev/null +++ b/packaging/vcpkg/ports/soatable/portfile.cmake @@ -0,0 +1,32 @@ +# Header-only port. Use this directory as a vcpkg overlay: +# vcpkg install soatable --overlay-ports=packaging/vcpkg/ports +# +# On each tagged release, replace SHA512 below with the value reported by: +# vcpkg hash +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO bbalouki/soatable + REF "v${VERSION}" + SHA512 0 + HEAD_REF main +) + +vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}" + OPTIONS + -DSOATABLE_BUILD_TESTS=OFF + -DSOATABLE_BUILD_EXAMPLES=OFF + -DSOATABLE_BUILD_BENCHMARKS=OFF +) + +vcpkg_cmake_install() + +vcpkg_cmake_config_fixup( + PACKAGE_NAME soatable + CONFIG_PATH "${CMAKE_INSTALL_LIBDIR}/cmake/soatable" +) + +# An interface library installs no binaries. +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug" "${CURRENT_PACKAGES_DIR}/lib") + +vcpkg_install_copyright(FILE_LIST "${SOURCE_PATH}/LICENSE") diff --git a/packaging/vcpkg/ports/soatable/vcpkg.json b/packaging/vcpkg/ports/soatable/vcpkg.json new file mode 100644 index 0000000..3f076c1 --- /dev/null +++ b/packaging/vcpkg/ports/soatable/vcpkg.json @@ -0,0 +1,17 @@ +{ + "name": "soatable", + "version": "0.3.0", + "description": "Header-only C++23 sparse Structure-of-Arrays table with generational handles.", + "homepage": "https://github.com/bbalouki/soatable", + "license": "MIT", + "dependencies": [ + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + } + ] +} diff --git a/scripts/amalgamate.py b/scripts/amalgamate.py new file mode 100644 index 0000000..d1ccc19 --- /dev/null +++ b/scripts/amalgamate.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Produce a drop-in single-header distribution of SoaTable. + +SoaTable is already a single header, so amalgamation simply stamps a provenance banner (version and +license pointer) onto a copy. The script exists so the published artifact is reproducible and so the +process generalizes if the library is ever split across multiple headers. + +Usage: + python scripts/amalgamate.py [--output dist/soatable.hpp] +""" + +from __future__ import annotations + +import argparse +import datetime +import pathlib +import re +import sys + +REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent +HEADER = REPO_ROOT / "include" / "soatable" / "soatable.hpp" + + +def read_version() -> str: + """Extract the project version from the top-level CMakeLists.txt.""" + text = (REPO_ROOT / "CMakeLists.txt").read_text(encoding="utf-8") + match = re.search(r"project\(\s*\w+\s+VERSION\s+([0-9]+\.[0-9]+\.[0-9]+)", text) + if match is None: + sys.exit("Could not determine project version from CMakeLists.txt") + return match.group(1) + + +def build_banner(version: str) -> str: + """Return the provenance comment prepended to the amalgamated header.""" + year = datetime.date.today().year + return ( + "// SoaTable - single-header amalgamation\n" + f"// Version: {version}\n" + f"// Generated: {datetime.date.today().isoformat()}\n" + "// Source: https://github.com/bbalouki/soatable\n" + f"// License: MIT (c) {year}. See LICENSE in the repository.\n" + "//\n" + "// This file is generated by scripts/amalgamate.py. Do not edit by hand.\n" + "\n" + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--output", + type=pathlib.Path, + default=REPO_ROOT / "dist" / "soatable.hpp", + help="Destination path for the amalgamated header.", + ) + args = parser.parse_args() + + if not HEADER.is_file(): + sys.exit(f"Header not found: {HEADER}") + + version = read_version() + content = build_banner(version) + HEADER.read_text(encoding="utf-8") + + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(content, encoding="utf-8") + print(f"Wrote {args.output} (SoaTable {version})") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_benchmarks.sh b/scripts/run_benchmarks.sh new file mode 100644 index 0000000..92bb458 --- /dev/null +++ b/scripts/run_benchmarks.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Build and run the SoaTable benchmark suite via the `bench` CMake preset, recording the toolchain +# and emitting a machine-readable results file. Google Benchmark prints CPU topology and cache sizes +# on startup, so the output is self-describing. +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${ROOT}" + +OUT_DIR="${ROOT}/bench" +mkdir -p "${OUT_DIR}" + +cmake --preset bench +cmake --build --preset bench + +BENCH_BIN="$(find "${ROOT}/build/bench" -name 'soatable_benchmark*' -type f -perm -u+x | head -n1)" +if [[ -z "${BENCH_BIN}" ]]; then + echo "Benchmark binary not found under build/bench" >&2 + exit 1 +fi + +# Record the build environment alongside the results so numbers stay reproducible/comparable. +{ + echo "date: $(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo "cmake: $(cmake --version | head -n1)" + echo "compiler: ${CXX:-default}" + echo "build_type: Release" +} > "${OUT_DIR}/environment.txt" + +"${BENCH_BIN}" \ + --benchmark_out="${OUT_DIR}/results.json" \ + --benchmark_out_format=json + +echo "Wrote ${OUT_DIR}/results.json and ${OUT_DIR}/environment.txt" From 5a3aadce03ea83e217d47b0699bb1c89742afee2 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Wed, 24 Jun 2026 15:56:16 +0100 Subject: [PATCH 05/58] Performance hardening (IMPROVEMENTS Phase 4) - 4.1 Broaden the benchmark suite beyond the single select scenario: insertion, erase-churn, single-column and parallel sort, a select density sweep, and Array-of-Structures plus hand-rolled columnar scan baselines. - 4.2 Back sort_by_column_parallel with a serial size threshold (parallel_sort_threshold) and let the calling thread reorder the last column so it overlaps the async tasks instead of idling; document the threading model and why a persistent thread pool is intentionally avoided. Add a large-table test that exercises the concurrent path above the threshold. - 4.3 Add a driver-heuristic validation benchmark comparing the automatic smallest-column driver against a forced largest-column driver. Measured (250k rows, Release): smallest-driver select ~168us vs forced largest-driver ~1547us (~9x), confirming the heuristic. 61/61 tests green. Co-Authored-By: Claude Opus 4.8 --- FEATURES.md | 9 +- README.md | 24 ++- benchmarks/soatable_benchmark.cpp | 264 ++++++++++++++++++++++++++---- include/soatable/soatable.hpp | 93 +++++++---- tests/unittests/sort_test.cpp | 27 +++ 5 files changed, 347 insertions(+), 70 deletions(-) diff --git a/FEATURES.md b/FEATURES.md index 7759e03..dc31cc4 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -111,13 +111,12 @@ layouts, and emerging **C++26 reflection**-based SoA generation. ## Phase C — Portability & interop (P1 / P2) -### C.1 C API + stable-ABI shim and language bindings +### C.1 Language bindings -- **What:** A C-linkage facade over a fixed set of column types, plus first-class **Python** bindings - (nanobind/pybind11) and a path for Rust / C# FFI. +- **What:** A First-class **Python** bindings (pybind11). - **Use cases:** This is the literal "numpy" forcing function — let data scientists `import soatable` - and get zero-copy numpy arrays via the buffer protocol; let Rust/C# embed the same store. -- **Sketch:** A C ABI exposing handle/column/select; a Python module mapping columns to numpy arrays + and get zero-copy numpy arrays via the buffer protocol. +- **Sketch:** Python module mapping columns to numpy arrays through A.1 spans + the buffer protocol. **Priority:** P1 · **Effort:** L ### C.2 Custom allocators diff --git a/README.md b/README.md index a666998..0243b28 100644 --- a/README.md +++ b/README.md @@ -235,12 +235,26 @@ cmake --build build-docs --target docs ## Benchmarks -Benchmark results (250,000 rows, selective join): +The benchmark suite covers insertion, erase-churn, single-column and parallel sort, sparse selection +across density sweeps, and the select driver heuristic, with Array-of-Structures and hand-rolled +columnar baselines. Run it (records the toolchain and emits a JSON results file): + +```sh +./scripts/run_benchmarks.sh # builds the `bench` preset and writes bench/results.json +``` + +Representative selective-join results (250,000 rows, Release): + | Method | Time | -|-------------------|----------------| -| SoaTable `select` | ~162,000 ns | -| AoS Branch Scan | ~2,035,000 ns | -_Run on 4x 2300 MHz CPU, Release Build._ +|-------------------------------------|----------------| +| SoaTable `select` (smallest driver) | ~168,000 ns | +| Forced largest-column driver | ~1,547,000 ns | +| Hand-rolled columnar scan | ~1,058,000 ns | +| AoS branch scan | ~1,303,000 ns | + +The ~9x gap between the smallest-driver and forced-largest-driver runs validates the heuristic of +driving iteration off the smallest required column. Numbers are machine dependent; see the harness +above to reproduce on your hardware. ## License diff --git a/benchmarks/soatable_benchmark.cpp b/benchmarks/soatable_benchmark.cpp index f2b0539..7e5c650 100644 --- a/benchmarks/soatable_benchmark.cpp +++ b/benchmarks/soatable_benchmark.cpp @@ -1,11 +1,17 @@ +// SoaTable benchmark suite. Covers the axes where regressions hide: insertion, erase-churn, +// single-column and parallel sort, sparse selection across density sweeps, and the select driver +// heuristic. Baselines compare against an Array-of-Structures scan and a hand-rolled columnar scan. + #include -#include +#include #include #include #include "soatable/soatable.hpp" +namespace { + struct Temperature { double celsius = 0.0; }; @@ -16,35 +22,128 @@ struct RegionId { std::uint32_t value = 0; }; -struct AOSRecord { - bool has_temperature = false; - bool has_pressure = false; - bool has_region = false; - Temperature temperature {}; - Pressure pressure {}; - RegionId region {}; -}; +using SensorTable = soatable::soa_table; -static void BM_SoaTableSelect(benchmark::State& state) { - const std::size_t row_count = static_cast(state.range(0)); - soatable::soa_table table; +// Deterministic per-row values so sort benchmarks do real work without consuming the presence RNG. +constexpr double temperature_value(std::size_t i) { + return static_cast((i * 48271U) % 1000U); +} +constexpr double pressure_value(std::size_t i) { + return static_cast((i * 16807U) % 1000U); +} + +// Build a table of the given size with the given per-column presence probabilities (fixed seed). +SensorTable build_table( + std::size_t row_count, double temp_p, double pressure_p, double region_p +) { + SensorTable table; table.reserve(row_count); std::mt19937_64 rng(0x5A0A7EULL); - std::bernoulli_distribution temp_present(0.70); - std::bernoulli_distribution pressure_present(0.40); - std::bernoulli_distribution region_present(0.05); + std::bernoulli_distribution temp_present(temp_p); + std::bernoulli_distribution pressure_present(pressure_p); + std::bernoulli_distribution region_present(region_p); for (std::size_t i = 0; i < row_count; ++i) { const auto row = table.insert(); - if (temp_present(rng)) - table.assign(row, 25.0); - if (pressure_present(rng)) - table.assign(row, 101.3); - if (region_present(rng)) - table.assign(row, 1); + if (temp_present(rng)) { + table.assign(row, Temperature {temperature_value(i)}); + } + if (pressure_present(rng)) { + table.assign(row, Pressure {pressure_value(i)}); + } + if (region_present(rng)) { + table.assign(row, RegionId {static_cast(i % 64U)}); + } + } + return table; +} + +} // namespace + +// --- Insertion -------------------------------------------------------------------------------- +static void BM_Insert(benchmark::State& state) { + const std::size_t row_count = static_cast(state.range(0)); + for (auto _ : state) { + SensorTable table; + table.reserve(row_count); + for (std::size_t i = 0; i < row_count; ++i) { + const auto row = table.insert(); + table.assign(row, Temperature {temperature_value(i)}); + } + benchmark::DoNotOptimize(table.size()); + } + state.SetItemsProcessed(state.iterations() * state.range(0)); +} +BENCHMARK(BM_Insert)->Arg(100'000); + +// --- Erase churn ------------------------------------------------------------------------------ +static void BM_EraseChurn(benchmark::State& state) { + const std::size_t row_count = static_cast(state.range(0)); + for (auto _ : state) { + SensorTable table; + std::vector ids; + ids.reserve(row_count); + table.reserve(row_count); + for (std::size_t i = 0; i < row_count; ++i) { + const auto row = table.insert(); + table.assign(row, Temperature {temperature_value(i)}); + ids.push_back(row); + } + for (const auto id : ids) { + table.erase(id); + } + benchmark::DoNotOptimize(table.size()); } + state.SetItemsProcessed(state.iterations() * state.range(0) * 2); +} +BENCHMARK(BM_EraseChurn)->Arg(100'000); +// --- Sorting ---------------------------------------------------------------------------------- +static void BM_SortByColumn(benchmark::State& state) { + const std::size_t row_count = static_cast(state.range(0)); + SensorTable table = build_table(row_count, 1.0, 1.0, 1.0); + bool ascending = true; + for (auto _ : state) { + if (ascending) { + table.sort_by_column( + [](const Temperature& a, const Temperature& b) { return a.celsius < b.celsius; } + ); + } else { + table.sort_by_column( + [](const Temperature& a, const Temperature& b) { return a.celsius > b.celsius; } + ); + } + ascending = !ascending; + benchmark::ClobberMemory(); + } +} +BENCHMARK(BM_SortByColumn)->Arg(100'000); + +static void BM_SortByColumnParallel(benchmark::State& state) { + const std::size_t row_count = static_cast(state.range(0)); + SensorTable table = build_table(row_count, 1.0, 1.0, 1.0); + bool ascending = true; + for (auto _ : state) { + if (ascending) { + table.sort_by_column_parallel( + [](const Temperature& a, const Temperature& b) { return a.celsius < b.celsius; } + ); + } else { + table.sort_by_column_parallel( + [](const Temperature& a, const Temperature& b) { return a.celsius > b.celsius; } + ); + } + ascending = !ascending; + benchmark::ClobberMemory(); + } +} +BENCHMARK(BM_SortByColumnParallel)->Arg(100'000); + +// --- Selection: SoaTable vs baselines --------------------------------------------------------- +static void BM_SoaTableSelect(benchmark::State& state) { + const std::size_t row_count = static_cast(state.range(0)); + SensorTable table = build_table(row_count, 0.70, 0.40, 0.05); for (auto _ : state) { double sum = 0.0; for (auto [id, temperature, pressure, region] : @@ -57,28 +156,36 @@ static void BM_SoaTableSelect(benchmark::State& state) { BENCHMARK(BM_SoaTableSelect)->Arg(100'000)->Arg(250'000); static void BM_AoSBranchScan(benchmark::State& state) { + struct AosRecord { + bool has_temperature = false; + bool has_pressure = false; + bool has_region = false; + double celsius = 0.0; + double kpa = 0.0; + std::uint32_t region = 0; + }; + const std::size_t row_count = static_cast(state.range(0)); - std::vector aos; + std::vector aos; aos.reserve(row_count); std::mt19937_64 rng(0x5A0A7EULL); std::bernoulli_distribution temp_present(0.70); std::bernoulli_distribution pressure_present(0.40); std::bernoulli_distribution region_present(0.05); - for (std::size_t i = 0; i < row_count; ++i) { - AOSRecord record; + AosRecord record; if (temp_present(rng)) { - record.has_temperature = true; - record.temperature.celsius = 25.0; + record.has_temperature = true; + record.celsius = temperature_value(i); } if (pressure_present(rng)) { record.has_pressure = true; - record.pressure.kpa = 101.3; + record.kpa = pressure_value(i); } if (region_present(rng)) { - record.has_region = true; - record.region.value = 1; + record.has_region = true; + record.region = static_cast(i % 64U); } aos.push_back(record); } @@ -87,7 +194,7 @@ static void BM_AoSBranchScan(benchmark::State& state) { double sum = 0.0; for (const auto& record : aos) { if (record.has_temperature && record.has_pressure && record.has_region) { - sum += record.temperature.celsius + record.pressure.kpa + record.region.value; + sum += record.celsius + record.kpa + record.region; } } benchmark::DoNotOptimize(sum); @@ -95,4 +202,101 @@ static void BM_AoSBranchScan(benchmark::State& state) { } BENCHMARK(BM_AoSBranchScan)->Arg(100'000)->Arg(250'000); +// --- Density sweep: select cost should track the smallest required column --------------------- +static void BM_SelectDensitySweep(benchmark::State& state) { + const std::size_t row_count = static_cast(state.range(0)); + const double region_density = static_cast(state.range(1)) / 100.0; + SensorTable table = build_table(row_count, 0.90, 0.90, region_density); + for (auto _ : state) { + double sum = 0.0; + for (auto [id, temperature, pressure, region] : + table.select()) { + sum += temperature.get().celsius + pressure.get().kpa + region.get().value; + } + benchmark::DoNotOptimize(sum); + } +} +// Same row count, increasing density of the driver column (1%, 5%, 20%, 50%). +BENCHMARK(BM_SelectDensitySweep) + ->Args({250'000, 1}) + ->Args({250'000, 5}) + ->Args({250'000, 20}) + ->Args({250'000, 50}); + +// --- Driver heuristic validation: smallest column should drive iteration ---------------------- +// RegionId is sparse and Temperature is dense; the auto driver picks RegionId. The forced variant +// drives off the dense Temperature column and filters manually, modelling a wrong driver choice. +static void BM_SelectAutoSmallestDriver(benchmark::State& state) { + const std::size_t row_count = static_cast(state.range(0)); + SensorTable table = build_table(row_count, 0.95, 0.95, 0.02); + for (auto _ : state) { + double sum = 0.0; + for (auto [id, temperature, pressure, region] : + table.select()) { + sum += temperature.get().celsius + pressure.get().kpa + region.get().value; + } + benchmark::DoNotOptimize(sum); + } +} +BENCHMARK(BM_SelectAutoSmallestDriver)->Arg(250'000); + +static void BM_SelectForcedLargestDriver(benchmark::State& state) { + const std::size_t row_count = static_cast(state.range(0)); + SensorTable table = build_table(row_count, 0.95, 0.95, 0.02); + for (auto _ : state) { + double sum = 0.0; + // Drive off the dense Temperature column, then filter the other columns by hand. + for (auto [id, temperature] : table.select()) { + const auto* pressure = table.try_get(id); + const auto* region = table.try_get(id); + if (pressure != nullptr && region != nullptr) { + sum += temperature.get().celsius + pressure->kpa + region->value; + } + } + benchmark::DoNotOptimize(sum); + } +} +BENCHMARK(BM_SelectForcedLargestDriver)->Arg(250'000); + +// --- Hand-rolled columnar (SoA) full scan baseline -------------------------------------------- +static void BM_HandRolledSoAScan(benchmark::State& state) { + const std::size_t row_count = static_cast(state.range(0)); + std::vector celsius(row_count); + std::vector kpa(row_count); + std::vector region(row_count); + std::vector has_temp(row_count, 0); + std::vector has_pres(row_count, 0); + std::vector has_region(row_count, 0); + + std::mt19937_64 rng(0x5A0A7EULL); + std::bernoulli_distribution temp_present(0.70); + std::bernoulli_distribution pressure_present(0.40); + std::bernoulli_distribution region_present(0.05); + for (std::size_t i = 0; i < row_count; ++i) { + if (temp_present(rng)) { + has_temp[i] = 1; + celsius[i] = temperature_value(i); + } + if (pressure_present(rng)) { + has_pres[i] = 1; + kpa[i] = pressure_value(i); + } + if (region_present(rng)) { + has_region[i] = 1; + region[i] = static_cast(i % 64U); + } + } + + for (auto _ : state) { + double sum = 0.0; + for (std::size_t i = 0; i < row_count; ++i) { + if (has_temp[i] != 0 && has_pres[i] != 0 && has_region[i] != 0) { + sum += celsius[i] + kpa[i] + region[i]; + } + } + benchmark::DoNotOptimize(sum); + } +} +BENCHMARK(BM_HandRolledSoAScan)->Arg(100'000)->Arg(250'000); + BENCHMARK_MAIN(); diff --git a/include/soatable/soatable.hpp b/include/soatable/soatable.hpp index 2ae9cc0..730449e 100644 --- a/include/soatable/soatable.hpp +++ b/include/soatable/soatable.hpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -692,6 +693,12 @@ class soa_table { static constexpr std::uint32_t npos = std::numeric_limits::max(); /// @brief Generation value at which a slot is retired instead of recycled (ABA wraparound bound). static constexpr std::uint32_t max_generation = std::numeric_limits::max(); + /// @brief Alive-row count below which sort_by_column_parallel falls back to a serial sort. + /// + /// Below this size the per-column task-launch overhead dominates the work saved, so the parallel + /// path is not worthwhile. The crossover is machine dependent; this is a conservative default + /// measured to favour the serial path for small tables. + static constexpr std::size_t parallel_sort_threshold = 4096; /// @brief Metadata for all row slots. std::vector m_rows; @@ -1303,11 +1310,18 @@ class soa_table { sort_by_column(std::forward(comp)); } - /// @brief Parallel version of sort_by_column. Reorders columns in parallel to speed up the + /// @brief Parallel version of sort_by_column. Reorders columns concurrently to speed up the /// process. /// @tparam T The type of the column to sort by. /// @tparam Compare The type of the comparator. /// @param comp The comparator to use for sorting values of type T. + /// @note Threading model: the row order is computed once on the calling thread, then each column + /// is reordered independently (columns are the unit of parallelism). The calling thread reorders + /// the last column itself while @c column_count-1 std::async tasks handle the rest, so the caller + /// is never idle. It falls back to a serial sort when there is a single column, a single hardware + /// thread, or fewer than @ref parallel_sort_threshold alive rows, where task-launch overhead + /// would dominate. A persistent thread pool is deliberately avoided: soa_table is a value-semantic + /// container, and owning worker threads would complicate its copy/move/lifetime semantics. /// @note Basic exception guarantee. An exception thrown while reordering a column is rethrown by /// future::get after all tasks join, leaving the table valid but partially reordered. template @@ -1315,37 +1329,56 @@ class soa_table { void sort_by_column_parallel(Compare&& comp) { if constexpr (column_count <= 1) { sort_by_column(std::forward(comp)); - return; - } - - auto& pool = std::get>(m_columns); - const auto& dense = pool.dense_rows(); - - std::vector sorted_rows = dense; - std::sort(sorted_rows.begin(), sorted_rows.end(), [&](std::uint32_t a, std::uint32_t b) { - return std::invoke(comp, pool.get(a), pool.get(b)); - }); - - std::vector> futures; - futures.reserve(column_count); + } else { + if (m_alive_count < parallel_sort_threshold || std::thread::hardware_concurrency() <= 1) { + sort_by_column(std::forward(comp)); + return; + } - std::apply( - [&](auto&... column_pool) { - (futures.push_back( - std::async( - std::launch::async, - [pool_ptr = &column_pool, &sorted_rows]() { - pool_ptr->reorder(sorted_rows); - } - ) - ), - ...); - }, - m_columns - ); + auto& pool = std::get>(m_columns); + const auto& dense = pool.dense_rows(); - for (auto& future : futures) { - future.get(); + std::vector sorted_rows = dense; + std::sort( + sorted_rows.begin(), sorted_rows.end(), + [&](std::uint32_t a, std::uint32_t b) { + return std::invoke(comp, pool.get(a), pool.get(b)); + } + ); + + std::vector> futures; + futures.reserve(column_count - 1); + + // Dispatch every column but the last as an async task, then reorder the last column on + // the calling thread so it overlaps the helper tasks instead of blocking idle. + std::size_t column_index = 0; + std::apply( + [&](auto&... column_pool) { + ( + [&] { + if (column_index == column_count - 1) { + column_pool.reorder(sorted_rows); + } else { + futures.push_back( + std::async( + std::launch::async, + [pool_ptr = &column_pool, &sorted_rows]() { + pool_ptr->reorder(sorted_rows); + } + ) + ); + } + ++column_index; + }(), + ... + ); + }, + m_columns + ); + + for (auto& future : futures) { + future.get(); + } } } diff --git a/tests/unittests/sort_test.cpp b/tests/unittests/sort_test.cpp index b9a2a59..1d24a8e 100644 --- a/tests/unittests/sort_test.cpp +++ b/tests/unittests/sort_test.cpp @@ -111,3 +111,30 @@ TEST(SortTest, ParallelSortHandlesSingleColumnTable) { } EXPECT_EQ(ages, (std::vector {1, 3})); } + +TEST(SortTest, ParallelSortLargeTableExercisesConcurrentPath) { + // Exceed parallel_sort_threshold so the genuine multi-column concurrent reorder runs, and verify + // it produces a fully sorted, internally consistent table. + constexpr int row_count = 20000; + DemoTable table; + table.reserve(row_count); + + for (int i = 0; i < row_count; ++i) { + const auto id = table.insert(); + table.assign(id, (row_count - i) % 997); // Deterministic, non-monotonic. + table.assign(id, "n"); + } + + table.sort_by_column_parallel([](const Age& a, const Age& b) { return a.value < b.value; }); + + int previous = -1; + int seen = 0; + for (auto [id, age, name] : table.select()) { + EXPECT_TRUE(table.is_valid(id)); + EXPECT_GE(age.get().value, previous); // Non-decreasing => sorted. + EXPECT_EQ(name.get().value, "n"); + previous = age.get().value; + ++seen; + } + EXPECT_EQ(seen, row_count); +} From 273f8edf212b1aab61cf9342b7daa5c5d16e58b6 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Wed, 24 Jun 2026 16:17:37 +0100 Subject: [PATCH 06/58] Foundational layout primitives: spans, validity bitmaps, serialization (FEATURES Phase A.1/A.3/A.4) - A.1 Zero-copy column spans: column() and a const overload return std::span over a column's dense values; row_indices() and make_row_id() map dense positions back to stable handles. This is the keystone the compute layer, bindings, and serialization build on. - A.3 Validity bitmaps: a packed Arrow-style bitmap type plus validity() reporting per-row column presence over the row-slot space, with for_each_set for branchless masked iteration and words() for interchange. - A.4 Serialization: opt-in with save()/load(), a versioned, schema-fingerprinted binary snapshot for trivially-copyable columns that round-trips handles, generations, the free list, and physical column order. A friend table_access keeps the core's public surface unchanged. New tests: span, validity, serialize (78 total). Verified: 78/78 ctest green; headers clean under clang -Wconversion -Wsign-conversion -Werror and MSVC /W4 /WX; save/load round-trip ASan+UBSan clean under churn + sort. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 14 ++ README.md | 3 + include/soatable/serialize.hpp | 292 +++++++++++++++++++++++++++++ include/soatable/soatable.hpp | 215 ++++++++++++++++++--- tests/CMakeLists.txt | 3 + tests/unittests/serialize_test.cpp | 110 +++++++++++ tests/unittests/span_test.cpp | 91 +++++++++ tests/unittests/validity_test.cpp | 84 +++++++++ tests/warning_check.cpp | 5 + 9 files changed, 787 insertions(+), 30 deletions(-) create mode 100644 include/soatable/serialize.hpp create mode 100644 tests/unittests/serialize_test.cpp create mode 100644 tests/unittests/span_test.cpp create mode 100644 tests/unittests/validity_test.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index 896960e..05ef900 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,20 @@ All notable changes to SoaTable are documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +Foundational layout primitives (FEATURES Phase A). + +### Added + +- `column()` (and a `const` overload) returning a `std::span` over a column's dense values, plus + `row_indices()` and `make_row_id()` to map dense positions back to stable handles (A.1). +- `validity()` returning an Arrow-style packed `bitmap` of per-row column presence, with + `for_each_set` for branchless masked iteration (A.3). +- Opt-in `` with `save()` / `load()`: a versioned, schema-checked binary + snapshot for trivially-copyable columns that round-trips handles, generations, and column order + (A.4). + ## [0.3.0] - 2026-06-24 API ergonomics and a deliberate naming migration (IMPROVEMENTS Phase 2). diff --git a/README.md b/README.md index 0243b28..4d35ec5 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,9 @@ In our benchmarks, SoaTable's `select` queries are often **5x to 15x faster** th - **Data Compression Utilities:** Includes `quantized_float`, `packed_bits`, and `delta_value` for compact storage. - **Reference-Semantic Row Views:** `view()` yields `row_view` proxies so structured bindings give real references (`for (auto [id, a, b] : table.view()) a.x += b.y;`) without `.get()`. - **Non-Throwing Accessors:** `get_expected()` returns `std::expected, access_error>` for hot paths and no-exceptions builds, alongside throwing `get()` and nullable `try_get()`. +- **Zero-Copy Column Spans:** `column()` hands a column's dense values to BLAS / a SIMD kernel / numpy with no copy; `row_indices()` maps them back to handles. +- **Validity Bitmaps:** `validity()` produces an Arrow-style packed presence bitmap for branchless masked iteration and interchange. +- **Serialization:** opt-in `` `save()` / `load()` snapshot trivially-copyable tables to a versioned, schema-checked byte buffer. ### Naming diff --git a/include/soatable/serialize.hpp b/include/soatable/serialize.hpp new file mode 100644 index 0000000..7264d8d --- /dev/null +++ b/include/soatable/serialize.hpp @@ -0,0 +1,292 @@ +#pragma once + +// Opt-in binary serialization for soa_table. Included separately so the core header pays nothing +// for it. Supports the trivially-copyable fast path: each column's dense values are dumped/loaded +// as raw bytes, wrapped in a versioned, schema-checked header that round-trips the full table state +// (handles, generations, free list, and physical column order). +// +// Portability: the format is self-describing and versioned, but values are written in the host's +// native byte order and ABI. It is intended for snapshots consumed by the same architecture (tick +// stores, simulation checkpoints). Cross-endian / cross-ABI interchange is a future enhancement and +// is the natural place for an Arrow bridge, built on column() spans and validity().words(). + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "soatable/soatable.hpp" + +namespace soatable { + +/// @brief Outcome of a load() call. +enum class serialize_status { + ok, + bad_magic, + version_mismatch, + schema_mismatch, + truncated, +}; + +namespace detail { + +/// @brief Friend accessor exposing soa_table's private state to the serializer only. +/// @tparam Columns The table's column types. +template +struct table_access { + using table_type = soa_table; + + static auto& rows(table_type& t) noexcept { return t.m_rows; } + static const auto& rows(const table_type& t) noexcept { return t.m_rows; } + static auto& free_links(table_type& t) noexcept { return t.m_free_links; } + static const auto& free_links(const table_type& t) noexcept { return t.m_free_links; } + static std::uint32_t& free_head(table_type& t) noexcept { return t.m_free_head; } + static std::uint32_t free_head(const table_type& t) noexcept { return t.m_free_head; } + static std::size_t& alive_count(table_type& t) noexcept { return t.m_alive_count; } + static std::size_t alive_count(const table_type& t) noexcept { return t.m_alive_count; } + static auto& columns(table_type& t) noexcept { return t.m_columns; } + static const auto& columns(const table_type& t) noexcept { return t.m_columns; } +}; + +inline constexpr std::array serialize_magic = { + std::byte {'S'}, + std::byte {'O'}, + std::byte {'A'}, + std::byte {'T'}, + std::byte {'B'}, + std::byte {'L'}, + std::byte {'E'}, + std::byte {'\0'} +}; +inline constexpr std::uint32_t serialize_version = 1; + +/// @brief FNV-1a hash of the column layout so load() can reject a mismatched schema. +template +constexpr std::uint64_t schema_fingerprint() { + std::uint64_t hash = 1469598103934665603ULL; + const auto mix = [&hash](std::uint64_t value) { + hash ^= value; + hash *= 1099511628211ULL; + }; + mix(sizeof...(Columns)); + (mix(sizeof(Columns)), ...); + (mix(alignof(Columns)), ...); + return hash; +} + +/// @brief Append a trivially-copyable value to a byte buffer. +template +void append_pod(std::vector& out, const T& value) { + const auto* bytes = reinterpret_cast(&value); + out.insert(out.end(), bytes, bytes + sizeof(T)); +} + +/// @brief Append raw bytes of a contiguous range of trivially-copyable values. +template +void append_array(std::vector& out, std::span values) { + const auto* bytes = reinterpret_cast(values.data()); + out.insert(out.end(), bytes, bytes + values.size_bytes()); +} + +/// @brief Sequential little-state reader over a byte span with bounds checking. +struct byte_reader { + std::span data; + std::size_t offset = 0; + bool ok = true; + + template + T read_pod() { + T value {}; + if (!ok || offset + sizeof(T) > data.size()) { + ok = false; + return value; + } + std::memcpy(&value, data.data() + offset, sizeof(T)); + offset += sizeof(T); + return value; + } + + template + void read_into(T* dst, std::size_t count) { + const std::size_t bytes = count * sizeof(T); + if (!ok || offset + bytes > data.size()) { + ok = false; + return; + } + std::memcpy(dst, data.data() + offset, bytes); + offset += bytes; + } +}; + +} // namespace detail + +/// @brief Serialize a table to a self-describing byte buffer (trivially-copyable columns only). +/// @tparam Columns The table's column types. +/// @param table The table to serialize. +/// @return A byte buffer that load() can reconstruct the table from. +template +[[nodiscard]] std::vector save(const soa_table& table) { + static_assert( + (std::is_trivially_copyable_v && ...), + "save() requires every column type to be trivially copyable." + ); + static_assert( + soa_table::column_count <= 64, + "save() supports up to 64 columns (the signature is stored as a 64-bit word)." + ); + + using access = detail::table_access; + std::vector out; + + const auto& rows = access::rows(table); + const auto& free_links = access::free_links(table); + + out.insert(out.end(), detail::serialize_magic.begin(), detail::serialize_magic.end()); + detail::append_pod(out, detail::serialize_version); + detail::append_pod(out, detail::schema_fingerprint()); + detail::append_pod(out, static_cast(rows.size())); + detail::append_pod(out, access::free_head(table)); + detail::append_pod(out, static_cast(access::alive_count(table))); + + for (const auto& meta : rows) { + detail::append_pod(out, meta.generation); + detail::append_pod(out, static_cast(meta.signature.to_ullong())); + detail::append_pod(out, static_cast(meta.alive ? 1 : 0)); + } + detail::append_array(out, std::span(free_links.data(), free_links.size())); + + // Each column: its dense row indices followed by its raw values, in declaration order. + std::apply( + [&out](const auto&... pool) { + ( + [&out](const auto& column_pool) { + using value_type = std::decay_t; + const auto& dense = column_pool.dense_rows(); + const auto& data = column_pool.raw_data(); + detail::append_pod(out, static_cast(dense.size())); + detail::append_array( + out, std::span(dense.data(), dense.size()) + ); + detail::append_array( + out, std::span(data.data(), data.size()) + ); + }(pool), + ...); + }, + access::columns(table) + ); + + return out; +} + +/// @brief Reconstruct a table from a buffer produced by save(). +/// @tparam Columns The table's column types (must match the schema that was saved). +/// @param table The table to overwrite (cleared first). +/// @param bytes The serialized buffer. +/// @return serialize_status::ok on success, or the reason the buffer was rejected. +template +[[nodiscard]] serialize_status load( + soa_table& table, std::span bytes +) { + static_assert( + (std::is_trivially_copyable_v && ...), + "load() requires every column type to be trivially copyable." + ); + static_assert( + soa_table::column_count <= 64, + "load() supports up to 64 columns (the signature is stored as a 64-bit word)." + ); + + using access = detail::table_access; + + detail::byte_reader reader {bytes}; + + std::array magic {}; + reader.read_into(magic.data(), magic.size()); + if (!reader.ok) { + return serialize_status::truncated; + } + if (magic != detail::serialize_magic) { + return serialize_status::bad_magic; + } + if (reader.read_pod() != detail::serialize_version) { + return serialize_status::version_mismatch; + } + if (reader.read_pod() != detail::schema_fingerprint()) { + return serialize_status::schema_mismatch; + } + + const auto row_slots = static_cast(reader.read_pod()); + const auto free_head = reader.read_pod(); + const auto alive_count = static_cast(reader.read_pod()); + if (!reader.ok) { + return serialize_status::truncated; + } + + table.clear(); + + auto& rows = access::rows(table); + rows.resize(row_slots); + for (auto& meta : rows) { + meta.generation = reader.read_pod(); + const auto sig = reader.read_pod(); + meta.signature = typename soa_table::signature_type {sig}; + meta.alive = reader.read_pod() != 0; + } + + auto& free_links = access::free_links(table); + free_links.resize(row_slots); + reader.read_into(free_links.data(), row_slots); + + access::free_head(table) = free_head; + access::alive_count(table) = alive_count; + if (!reader.ok) { + table.clear(); + return serialize_status::truncated; + } + + // Replay each column's dense entries through emplace, rebuilding the sparse index. + bool columns_ok = true; + std::apply( + [&](auto&... pool) { + ( + [&](auto& column_pool) { + if (!columns_ok) { + return; + } + using value_type = std::decay_t; + column_pool.clear(); + const auto count = static_cast(reader.read_pod()); + if (!reader.ok) { + columns_ok = false; + return; + } + std::vector dense(count); + reader.read_into(dense.data(), count); + std::vector data(count); + reader.read_into(data.data(), count); + if (!reader.ok) { + columns_ok = false; + return; + } + for (std::size_t i = 0; i < count; ++i) { + column_pool.emplace(dense[i], data[i]); + } + }(pool), + ...); + }, + access::columns(table) + ); + + if (!columns_ok) { + table.clear(); + return serialize_status::truncated; + } + return serialize_status::ok; +} + +} // namespace soatable diff --git a/include/soatable/soatable.hpp b/include/soatable/soatable.hpp index 730449e..c780eb5 100644 --- a/include/soatable/soatable.hpp +++ b/include/soatable/soatable.hpp @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -13,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -122,6 +124,12 @@ struct index_of> { /// @brief Detail namespace for internal implementation details. namespace detail { +/// @brief Friend accessor granting the opt-in header access to the private +/// state of a soa_table without widening its public surface. Defined only in that header. +/// @tparam Columns The table's column types. +template +struct table_access; + /// @brief Generates a mask with the specified number of low bits set. /// @tparam UInt The unsigned integer type. /// @param bits The number of bits to set. @@ -390,6 +398,78 @@ class dirty_mask { [[nodiscard]] constexpr MaskType get_mask() const { return m_mask; } }; +// ========================= +// 2b. Validity bitmap +// ========================= + +/// @brief A packed bit container for per-row column presence (Arrow-style validity). +/// +/// Bits are stored little-endian within 64-bit words so the raw words() can be handed to an +/// Arrow-compatible consumer. Produced by soa_table::validity(); see that method for semantics. +class bitmap { + public: + /// @brief The word type backing the bitmap. + using word_type = std::uint64_t; + /// @brief Number of bits per backing word. + static constexpr std::size_t bits_per_word = std::numeric_limits::digits; + + /// @brief Construct an all-clear bitmap of the given bit length. + /// @param bit_count The number of bits the bitmap addresses. + explicit bitmap(std::size_t bit_count) + : m_words((bit_count + bits_per_word - 1) / bits_per_word, word_type {0}), + m_size(bit_count) {} + + /// @brief Set the bit at the given position. + /// @param index The bit position. + void set(std::size_t index) { + m_words[index / bits_per_word] |= word_type {1} << (index % bits_per_word); + } + + /// @brief Test the bit at the given position. + /// @param index The bit position. + /// @return True if the bit is set, false otherwise. + [[nodiscard]] bool test(std::size_t index) const { + return (m_words[index / bits_per_word] >> (index % bits_per_word) & word_type {1}) != 0; + } + + /// @brief Number of bits the bitmap addresses. + [[nodiscard]] std::size_t size() const noexcept { return m_size; } + + /// @brief Number of set bits. + [[nodiscard]] std::size_t count() const { + std::size_t total = 0; + for (const word_type word : m_words) { + total += static_cast(std::popcount(word)); + } + return total; + } + + /// @brief The raw backing words (little-endian bit order), suitable for Arrow interchange. + [[nodiscard]] std::span words() const noexcept { + return std::span(m_words.data(), m_words.size()); + } + + /// @brief Invoke a callback for each set bit position, in ascending order, branch-free per + /// word. + /// @tparam Func A callable accepting a std::size_t bit position. + /// @param func The callback. + template + void for_each_set(Func&& func) const { + for (std::size_t word_index = 0; word_index < m_words.size(); ++word_index) { + word_type remaining = m_words[word_index]; + while (remaining != 0) { + const auto bit = static_cast(std::countr_zero(remaining)); + std::invoke(func, word_index * bits_per_word + bit); + remaining &= remaining - 1; // Clear the lowest set bit. + } + } + } + + private: + std::vector m_words; + std::size_t m_size = 0; +}; + // ========================= // 3. Sparse column storage // ========================= @@ -493,8 +573,9 @@ class column_vector { /// @param args Arguments to pass to the constructor of T. /// @return Reference to the emplaced value. /// @note Strong exception guarantee when inserting a new value: the dense storage is grown and - /// only committed through the sparse slot once every throwing step has succeeded. Overwriting an - /// existing value offers the guarantee of T's assignment (strong for nothrow-assignable types). + /// only committed through the sparse slot once every throwing step has succeeded. Overwriting + /// an existing value offers the guarantee of T's assignment (strong for nothrow-assignable + /// types). template T& emplace(std::uint32_t row_index, Args&&... args) { if (row_index >= m_sparse.size()) { @@ -595,7 +676,8 @@ class column_vector { /// row.get().x += row.get().x; /// } /// @endcode -/// A row_view is invalidated by the same operations that invalidate column references (erase, sort). +/// A row_view is invalidated by the same operations that invalidate column references (erase, +/// sort). template class row_view { public: @@ -691,13 +773,14 @@ class soa_table { private: static constexpr std::uint32_t npos = std::numeric_limits::max(); - /// @brief Generation value at which a slot is retired instead of recycled (ABA wraparound bound). + /// @brief Generation value at which a slot is retired instead of recycled (ABA wraparound + /// bound). static constexpr std::uint32_t max_generation = std::numeric_limits::max(); /// @brief Alive-row count below which sort_by_column_parallel falls back to a serial sort. /// - /// Below this size the per-column task-launch overhead dominates the work saved, so the parallel - /// path is not worthwhile. The crossover is machine dependent; this is a conservative default - /// measured to favour the serial path for small tables. + /// Below this size the per-column task-launch overhead dominates the work saved, so the + /// parallel path is not worthwhile. The crossover is machine dependent; this is a conservative + /// default measured to favour the serial path for small tables. static constexpr std::size_t parallel_sort_threshold = 4096; /// @brief Metadata for all row slots. @@ -711,6 +794,9 @@ class soa_table { /// @brief Storage for each column. std::tuple...> m_columns; + /// @brief Grants the opt-in serialization header access to the private state above. + friend struct detail::table_access; + /// @brief Internal check for index overflow. [[nodiscard]] static constexpr bool can_address_rows(std::size_t count) { return count <= std::numeric_limits::max(); @@ -829,11 +915,12 @@ class soa_table { /// @brief Erase a row by ID. /// @param id The row_id of the row to erase. /// @note Basic exception guarantee: column element removal relies on T's move assignment, which - /// may throw for non-nothrow-movable types, leaving the table valid but in an unspecified state. - /// @note ABA safety: a slot's generation defeats stale handles for up to 2^32 erase/reuse cycles. - /// When that counter would wrap, the slot is retired (never recycled) so a saturated stale handle - /// can never alias a freshly inserted row. Retiring leaks one slot, which is acceptable given the - /// cycle count required to reach it. + /// may throw for non-nothrow-movable types, leaving the table valid but in an unspecified + /// state. + /// @note ABA safety: a slot's generation defeats stale handles for up to 2^32 erase/reuse + /// cycles. When that counter would wrap, the slot is retired (never recycled) so a saturated + /// stale handle can never alias a freshly inserted row. Retiring leaks one slot, which is + /// acceptable given the cycle count required to reach it. void erase(row_id id) { if (!is_valid(id)) { return; @@ -869,8 +956,9 @@ class soa_table { /// @param id The row_id of the row. /// @param args Arguments to pass to the constructor of T. /// @return Reference to the assigned value. - /// @note Strong exception guarantee when adding a new column value (delegates to the strong path - /// of column_vector::emplace); overwriting an existing value inherits T's assignment guarantee. + /// @note Strong exception guarantee when adding a new column value (delegates to the strong + /// path of column_vector::emplace); overwriting an existing value inherits T's assignment + /// guarantee. /// @throws std::out_of_range If the row_id is invalid (thrown before any mutation). template requires registered_column_v @@ -1196,6 +1284,72 @@ class soa_table { }); } + /// @brief Zero-copy view of a column's dense values as a contiguous span. + /// @tparam T The column type. + /// @return A span over the column's packed values, ready to hand to BLAS / a SIMD kernel / + /// numpy. + /// @note The span covers only rows that currently have the column, in dense storage order; pair + /// it with row_indices() to recover each value's row. Any operation that adds, removes, or + /// reorders the column (assign, unassign, erase, sort, clear, reserve) invalidates the span. + template + requires registered_column_v + [[nodiscard]] std::span column() { + auto& data = std::get>(m_columns).raw_data(); + return std::span(data.data(), data.size()); + } + + /// @brief Const zero-copy view of a column's dense values. + /// @tparam T The column type. + /// @return A span over the column's packed const values. + template + requires registered_column_v + [[nodiscard]] std::span column() const { + const auto& data = std::get>(m_columns).raw_data(); + return std::span(data.data(), data.size()); + } + + /// @brief Row indices parallel to column(), mapping each dense value back to its row. + /// @tparam T The column type. + /// @return A span where element i is the internal row index of column()[i]. + /// @note Combine an index with make_row_id() to obtain a stable handle. Invalidated by the same + /// operations as column(). + template + requires registered_column_v + [[nodiscard]] std::span row_indices() const { + const auto& dense = std::get>(m_columns).dense_rows(); + return std::span(dense.data(), dense.size()); + } + + /// @brief Recover a stable handle from an internal row index (e.g. from row_indices()). + /// @param row_index The internal row index. + /// @return The row_id for that slot, or an invalid handle if the index is out of range or dead. + [[nodiscard]] row_id make_row_id(std::uint32_t row_index) const noexcept { + if (row_index >= m_rows.size() || !m_rows[row_index].alive) { + return row_id {npos, 0}; + } + return row_id {row_index, m_rows[row_index].generation}; + } + + /// @brief Arrow-style validity bitmap of a column over the row-slot space. + /// @tparam T The column type. + /// @return A bitmap of length row_slots() where bit i is set iff slot i is alive and has T. + /// @note Indexed by internal row index (row_id::index), so it composes with make_row_id() and + /// the raw word view feeds branchless masked iteration or Arrow interchange. Rebuilt on each + /// call. + template + requires registered_column_v + [[nodiscard]] bitmap validity() const { + constexpr std::size_t column_index = index_of>::value; + bitmap mask(m_rows.size()); + for (std::size_t row_index = 0; row_index < m_rows.size(); ++row_index) { + const auto& meta = m_rows[row_index]; + if (meta.alive && meta.signature.test(column_index)) { + mask.set(row_index); + } + } + return mask; + } + /// @brief Fast batch insertion of multiple rows. /// @param count The number of rows to insert. /// @return A vector of newly created row_ids. @@ -1231,8 +1385,9 @@ class soa_table { /// criteria. /// @tparam SortCriteria The types of the comparison criteria. /// @param criteria The comparison criteria. Each should be a pair of (ColumnType, Comparator). - /// @note Basic exception guarantee: each column is reordered in turn (see column_vector::reorder), - /// so a throwing element move leaves the table valid but only partially reordered. + /// @note Basic exception guarantee: each column is reordered in turn (see + /// column_vector::reorder), so a throwing element move leaves the table valid but only + /// partially reordered. template void sort_by_multi(SortCriteria&&... criteria) { // Find smallest driver if possible, otherwise use all alive rows @@ -1315,22 +1470,24 @@ class soa_table { /// @tparam T The type of the column to sort by. /// @tparam Compare The type of the comparator. /// @param comp The comparator to use for sorting values of type T. - /// @note Threading model: the row order is computed once on the calling thread, then each column - /// is reordered independently (columns are the unit of parallelism). The calling thread reorders - /// the last column itself while @c column_count-1 std::async tasks handle the rest, so the caller - /// is never idle. It falls back to a serial sort when there is a single column, a single hardware - /// thread, or fewer than @ref parallel_sort_threshold alive rows, where task-launch overhead - /// would dominate. A persistent thread pool is deliberately avoided: soa_table is a value-semantic - /// container, and owning worker threads would complicate its copy/move/lifetime semantics. - /// @note Basic exception guarantee. An exception thrown while reordering a column is rethrown by - /// future::get after all tasks join, leaving the table valid but partially reordered. + /// @note Threading model: the row order is computed once on the calling thread, then each + /// column is reordered independently (columns are the unit of parallelism). The calling thread + /// reorders the last column itself while @c column_count-1 std::async tasks handle the rest, so + /// the caller is never idle. It falls back to a serial sort when there is a single column, a + /// single hardware thread, or fewer than @ref parallel_sort_threshold alive rows, where + /// task-launch overhead would dominate. A persistent thread pool is deliberately avoided: + /// soa_table is a value-semantic container, and owning worker threads would complicate its + /// copy/move/lifetime semantics. + /// @note Basic exception guarantee. An exception thrown while reordering a column is rethrown + /// by future::get after all tasks join, leaving the table valid but partially reordered. template requires registered_column_v void sort_by_column_parallel(Compare&& comp) { if constexpr (column_count <= 1) { sort_by_column(std::forward(comp)); } else { - if (m_alive_count < parallel_sort_threshold || std::thread::hardware_concurrency() <= 1) { + if (m_alive_count < parallel_sort_threshold || + std::thread::hardware_concurrency() <= 1) { sort_by_column(std::forward(comp)); return; } @@ -1340,8 +1497,7 @@ class soa_table { std::vector sorted_rows = dense; std::sort( - sorted_rows.begin(), sorted_rows.end(), - [&](std::uint32_t a, std::uint32_t b) { + sorted_rows.begin(), sorted_rows.end(), [&](std::uint32_t a, std::uint32_t b) { return std::invoke(comp, pool.get(a), pool.get(b)); } ); @@ -1370,8 +1526,7 @@ class soa_table { } ++column_index; }(), - ... - ); + ...); }, m_columns ); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index dbb031a..964b1a1 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -38,6 +38,9 @@ set(SOATABLE_UNIT_TEST_SOURCES unittests/invariant_test.cpp unittests/view_test.cpp unittests/expected_test.cpp + unittests/span_test.cpp + unittests/validity_test.cpp + unittests/serialize_test.cpp unittests/const_correctness_test.cpp unittests/deprecated_alias_test.cpp unittests/include_hygiene_test.cpp diff --git a/tests/unittests/serialize_test.cpp b/tests/unittests/serialize_test.cpp new file mode 100644 index 0000000..f384caf --- /dev/null +++ b/tests/unittests/serialize_test.cpp @@ -0,0 +1,110 @@ +// A.4 serialization: round-trip a table through save()/load(), and reject corrupt or mismatched +// buffers. Uses trivially-copyable columns, which the fast path requires. + +#include + +#include +#include +#include + +#include "soatable/serialize.hpp" +#include "soatable/soatable.hpp" + +namespace { +struct Px { + double value = 0.0; +}; +struct Qty { + int value = 0; +}; +struct Extra { + long value = 0; +}; +using TickTable = soatable::soa_table; +} // namespace + +TEST(SerializeTest, RoundTripPreservesValuesAndHandles) { + TickTable table; + const auto a = table.insert(); + table.assign(a, Px {10.5}); + table.assign(a, Qty {3}); + const auto b = table.insert(); + table.assign(b, Px {20.0}); // No Qty. + + const std::vector bytes = soatable::save(table); + + TickTable restored; + ASSERT_EQ(soatable::load(restored, bytes), soatable::serialize_status::ok); + + EXPECT_EQ(restored.size(), table.size()); + ASSERT_TRUE(restored.is_valid(a)); + ASSERT_TRUE(restored.is_valid(b)); + EXPECT_DOUBLE_EQ(restored.get(a).value, 10.5); + EXPECT_EQ(restored.get(a).value, 3); + EXPECT_DOUBLE_EQ(restored.get(b).value, 20.0); + EXPECT_FALSE(restored.contains(b)); +} + +TEST(SerializeTest, RoundTripPreservesGenerationsAfterErase) { + TickTable table; + const auto first = table.insert(); + table.erase(first); // Bumps the slot's generation. + const auto second = table.insert(); + table.assign(second, Px {1.0}); + + const std::vector bytes = soatable::save(table); + TickTable restored; + ASSERT_EQ(soatable::load(restored, bytes), soatable::serialize_status::ok); + + // A stale handle must remain invalid after a round trip; the live one valid. + EXPECT_FALSE(restored.is_valid(first)); + EXPECT_TRUE(restored.is_valid(second)); +} + +TEST(SerializeTest, RoundTripPreservesPhysicalOrderAfterSort) { + TickTable table; + for (int i = 0; i < 8; ++i) { + const auto id = table.insert(); + table.assign(id, Px {static_cast((i * 5) % 7)}); + } + table.sort_by_column([](const Px& l, const Px& r) { return l.value < r.value; }); + + const std::vector bytes = soatable::save(table); + TickTable restored; + ASSERT_EQ(soatable::load(restored, bytes), soatable::serialize_status::ok); + + const std::span original = table.column(); + const std::span reloaded = restored.column(); + ASSERT_EQ(original.size(), reloaded.size()); + for (std::size_t i = 0; i < original.size(); ++i) { + EXPECT_DOUBLE_EQ(original[i].value, reloaded[i].value); + } +} + +TEST(SerializeTest, RejectsBadMagic) { + std::vector garbage(64, std::byte {0x7F}); + TickTable table; + EXPECT_EQ(soatable::load(table, garbage), soatable::serialize_status::bad_magic); +} + +TEST(SerializeTest, RejectsTruncatedBuffer) { + TickTable table; + const auto id = table.insert(); + table.assign(id, Px {1.0}); + + std::vector bytes = soatable::save(table); + bytes.resize(bytes.size() / 2); // Chop the buffer in half. + + TickTable restored; + EXPECT_EQ(soatable::load(restored, bytes), soatable::serialize_status::truncated); +} + +TEST(SerializeTest, RejectsSchemaMismatch) { + TickTable table; + table.insert(); + const std::vector bytes = soatable::save(table); + + // A different schema (extra column) must be rejected. + soatable::soa_table other; + EXPECT_EQ(soatable::load(other, bytes), soatable::serialize_status::schema_mismatch); +} diff --git a/tests/unittests/span_test.cpp b/tests/unittests/span_test.cpp new file mode 100644 index 0000000..b240abf --- /dev/null +++ b/tests/unittests/span_test.cpp @@ -0,0 +1,91 @@ +// A.1 zero-copy column spans: column() exposes dense values contiguously, row_indices() maps +// them back to rows, and mutations through the span are visible in the table. + +#include + +#include +#include +#include + +#include "soatable/soatable.hpp" +#include "test_types.hpp" + +using soatable_test::Age; +using soatable_test::DemoTable; +using soatable_test::Name; + +TEST(SpanTest, ColumnSpanCoversDenseValues) { + DemoTable table; + const auto a = table.insert(); + table.assign(a, 1); + const auto b = table.insert(); + table.assign(b, 2); + const auto c = table.insert(); // No Age, so excluded from the span. + static_cast(c); + + const std::span ages = table.column(); + EXPECT_EQ(ages.size(), 2U); + + const int sum = std::accumulate( + ages.begin(), ages.end(), 0, [](int acc, const Age& age) { return acc + age.value; } + ); + EXPECT_EQ(sum, 3); +} + +TEST(SpanTest, MutationThroughSpanIsVisibleInTable) { + DemoTable table; + const auto id = table.insert(); + table.assign(id, 10); + + std::span ages = table.column(); + ASSERT_EQ(ages.size(), 1U); + ages[0].value = 99; + + EXPECT_EQ(table.get(id).value, 99); +} + +TEST(SpanTest, RowIndicesMapValuesBackToHandles) { + DemoTable table; + const auto a = table.insert(); + table.assign(a, 5); + const auto b = table.insert(); + table.assign(b, 6); + + const std::span ages = table.column(); + const std::span indices = table.row_indices(); + ASSERT_EQ(ages.size(), indices.size()); + + for (std::size_t i = 0; i < ages.size(); ++i) { + const soatable::row_id id = table.make_row_id(indices[i]); + ASSERT_TRUE(table.is_valid(id)); + EXPECT_EQ(table.get(id).value, ages[i].value); + } +} + +TEST(SpanTest, EmptyColumnYieldsEmptySpan) { + DemoTable table; + const auto id = table.insert(); + table.assign(id, "x"); + + EXPECT_TRUE(table.column().empty()); + EXPECT_TRUE(table.row_indices().empty()); +} + +TEST(SpanTest, ConstTableYieldsConstSpan) { + DemoTable table; + const auto id = table.insert(); + table.assign(id, 7); + + const DemoTable& const_table = table; + const std::span ages = const_table.column(); + ASSERT_EQ(ages.size(), 1U); + EXPECT_EQ(ages[0].value, 7); +} + +TEST(SpanTest, MakeRowIdRejectsDeadIndex) { + DemoTable table; + const auto id = table.insert(); + table.erase(id); + + EXPECT_FALSE(table.is_valid(table.make_row_id(id.index))); +} diff --git a/tests/unittests/validity_test.cpp b/tests/unittests/validity_test.cpp new file mode 100644 index 0000000..f4373fa --- /dev/null +++ b/tests/unittests/validity_test.cpp @@ -0,0 +1,84 @@ +// A.3 validity bitmaps: validity() reports per-row column presence, composes with make_row_id, +// and drives branchless masked iteration via for_each_set. + +#include + +#include +#include + +#include "soatable/soatable.hpp" +#include "test_types.hpp" + +using soatable_test::Age; +using soatable_test::DemoTable; +using soatable_test::Name; + +TEST(ValidityTest, BitmapMarksRowsThatHaveColumn) { + DemoTable table; + const auto a = table.insert(); + table.assign(a, 1); + const auto b = table.insert(); // No Age. + const auto c = table.insert(); + table.assign(c, 3); + + const soatable::bitmap mask = table.validity(); + EXPECT_EQ(mask.size(), table.row_slots()); + EXPECT_EQ(mask.count(), 2U); + EXPECT_TRUE(mask.test(a.index)); + EXPECT_FALSE(mask.test(b.index)); + EXPECT_TRUE(mask.test(c.index)); +} + +TEST(ValidityTest, ErasedRowsAreNotValid) { + DemoTable table; + const auto a = table.insert(); + table.assign(a, 1); + const auto b = table.insert(); + table.assign(b, 2); + table.erase(a); + + const soatable::bitmap mask = table.validity(); + EXPECT_FALSE(mask.test(a.index)); + EXPECT_TRUE(mask.test(b.index)); + EXPECT_EQ(mask.count(), 1U); +} + +TEST(ValidityTest, ForEachSetVisitsPresentRowsInOrder) { + DemoTable table; + std::vector expected; + for (int i = 0; i < 5; ++i) { + const auto id = table.insert(); + if (i % 2 == 0) { + table.assign(id, i); + expected.push_back(id.index); + } + } + + std::vector visited; + table.validity().for_each_set([&](std::size_t bit) { + visited.push_back(static_cast(bit)); + }); + EXPECT_EQ(visited, expected); +} + +TEST(ValidityTest, BitmapComposesWithMakeRowId) { + DemoTable table; + const auto a = table.insert(); + table.assign(a, 42); + + int matched = 0; + table.validity().for_each_set([&](std::size_t bit) { + const auto id = table.make_row_id(static_cast(bit)); + ASSERT_TRUE(table.is_valid(id)); + EXPECT_EQ(table.get(id).value, 42); + ++matched; + }); + EXPECT_EQ(matched, 1); +} + +TEST(ValidityTest, EmptyColumnHasZeroCount) { + DemoTable table; + const auto id = table.insert(); + table.assign(id, "x"); + EXPECT_EQ(table.validity().count(), 0U); +} diff --git a/tests/warning_check.cpp b/tests/warning_check.cpp index 8d08a12..50fcfce 100644 --- a/tests/warning_check.cpp +++ b/tests/warning_check.cpp @@ -28,6 +28,11 @@ int main() { total += found->get().value; } + for (const Position& position : table.column()) { + total += position.value; + } + total += static_cast(table.row_indices().size()); + table.sort_by_column( [](const Position& lhs, const Position& rhs) { return lhs.value < rhs.value; } ); From 083843216f6a7f6b56ce4bebb0805c7a37ffc8dd Mon Sep 17 00:00:00 2001 From: bbalouki Date: Wed, 24 Jun 2026 16:28:02 +0100 Subject: [PATCH 07/58] SIMD-aligned column storage (FEATURES Phase A.2, alignment half) Over-align dense column storage to simd_alignment (64 bytes, matching Apache Arrow) through an internal aligned_allocator, so column() spans are ready for aligned SIMD loads and are cache-line friendly. column_vector now stores its data in an aligned_vector; raw_data() and reorder use the same aligned type. The optional AoSoA tiled-layout policy (the other half of A.2) is intentionally deferred as a separate, larger change. Added a runtime alignment assertion. Verified: 79/79 ctest green; clean under clang -Wconversion -Wsign-conversion -Werror and MSVC /W4 /WX; ASan+UBSan clean under churn + sort (no aligned new/delete mismatch). Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 3 ++ include/soatable/soatable.hpp | 77 ++++++++++++++++++++++++++++++++--- tests/unittests/span_test.cpp | 13 ++++++ 3 files changed, 88 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 05ef900..676a22a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,9 @@ Foundational layout primitives (FEATURES Phase A). - Opt-in `` with `save()` / `load()`: a versioned, schema-checked binary snapshot for trivially-copyable columns that round-trips handles, generations, and column order (A.4). +- SIMD-aligned column storage: dense column data is over-aligned to `simd_alignment` (64 bytes, + matching Apache Arrow) via an internal aligned allocator, so `column()` spans are ready for + aligned SIMD loads (A.2, alignment half). The optional AoSoA tiled layout remains pending. ## [0.3.0] - 2026-06-24 diff --git a/include/soatable/soatable.hpp b/include/soatable/soatable.hpp index c780eb5..d052377 100644 --- a/include/soatable/soatable.hpp +++ b/include/soatable/soatable.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -121,6 +122,18 @@ struct index_of> { ); }; +/// @brief Default over-alignment (in bytes) for dense column storage. +/// +/// Columns are over-aligned to this boundary (matching Apache Arrow's 64-byte recommendation) so the +/// spans returned by soa_table::column() are ready for aligned SIMD loads and cache-line friendly. +inline constexpr std::size_t simd_alignment = 64; + +/// @brief The alignment used for a column of T: the larger of T's natural alignment and simd_alignment. +/// @tparam T The column value type. +template +inline constexpr std::size_t column_alignment = + (alignof(T) > simd_alignment) ? alignof(T) : simd_alignment; + /// @brief Detail namespace for internal implementation details. namespace detail { @@ -130,6 +143,58 @@ namespace detail { template struct table_access; +/// @brief A standard-conforming allocator that over-aligns its allocations to Alignment bytes. +/// @tparam T The allocated value type. +/// @tparam Alignment The required alignment in bytes (defaults to column_alignment). +template > +struct aligned_allocator { + static_assert(Alignment >= alignof(T), "Alignment must be at least alignof(T)."); + + /// @brief The allocated value type. + using value_type = T; + /// @brief The over-alignment in bytes. + static constexpr std::size_t alignment = Alignment; + + /// @brief Rebind helper so container nodes can be allocated with the same alignment policy. + template + struct rebind { + using other = aligned_allocator>; + }; + + constexpr aligned_allocator() noexcept = default; + + /// @brief Converting constructor required by the Allocator concept. + template + constexpr aligned_allocator(const aligned_allocator&) noexcept {} + + /// @brief Allocate n over-aligned elements. + /// @param count The number of elements. + /// @return Pointer to the aligned storage. + [[nodiscard]] T* allocate(std::size_t count) { + if (count > std::numeric_limits::max() / sizeof(T)) { + throw std::bad_alloc(); + } + void* ptr = ::operator new(count * sizeof(T), std::align_val_t {Alignment}); + return static_cast(ptr); + } + + /// @brief Release storage obtained from allocate(). + /// @param ptr The pointer to release. + void deallocate(T* ptr, std::size_t) noexcept { + ::operator delete(ptr, std::align_val_t {Alignment}); + } + + template + constexpr bool operator==(const aligned_allocator&) const noexcept { + return Alignment == OtherAlignment; + } +}; + +/// @brief A vector whose storage is over-aligned to column_alignment for SIMD-friendly columns. +/// @tparam T The element type. +template +using aligned_vector = std::vector>; + /// @brief Generates a mask with the specified number of low bits set. /// @tparam UInt The unsigned integer type. /// @param bits The number of bits to set. @@ -487,6 +552,8 @@ class column_vector { /// Widening this to `std::uint64_t` is the single change required for a 64-bit build; every /// internal index derives from it. Kept at 32 bits to match `row_id::index`. using size_type = std::uint32_t; + /// @brief The dense storage type: a vector over-aligned to column_alignment for SIMD loads. + using data_vector = detail::aligned_vector; private: /// @brief Sentinel stored in m_sparse for a row that has no value in this column. @@ -495,8 +562,8 @@ class column_vector { std::vector m_sparse; /// @brief Maps dense_index -> row_index. std::vector m_dense; - /// @brief The actual data stored densely. - std::vector m_data; + /// @brief The actual data stored densely (over-aligned for SIMD-friendly column spans). + data_vector m_data; public: /// @brief Default constructor. @@ -625,7 +692,7 @@ class column_vector { /// column is left in a valid but unspecified state. The index rebuild after the move loop is /// no-throw, so the sparse/dense/data arrays are never left mutually inconsistent. void reorder(const std::vector& row_order) { - std::vector new_data; + data_vector new_data; std::vector new_dense; new_data.reserve(m_data.size()); new_dense.reserve(m_dense.size()); @@ -654,9 +721,9 @@ class column_vector { /// @brief Get the list of row indices that have values in this column. [[nodiscard]] const std::vector& dense_rows() const { return m_dense; } /// @brief Access the raw dense data. - std::vector& raw_data() { return m_data; } + data_vector& raw_data() { return m_data; } /// @brief Access the raw dense data (const). - const std::vector& raw_data() const { return m_data; } + const data_vector& raw_data() const { return m_data; } }; // =================================== diff --git a/tests/unittests/span_test.cpp b/tests/unittests/span_test.cpp index b240abf..2060ae1 100644 --- a/tests/unittests/span_test.cpp +++ b/tests/unittests/span_test.cpp @@ -82,6 +82,19 @@ TEST(SpanTest, ConstTableYieldsConstSpan) { EXPECT_EQ(ages[0].value, 7); } +TEST(SpanTest, ColumnStorageIsSimdOverAligned) { + // A.2: column storage is over-aligned (Arrow's 64-byte recommendation) for aligned SIMD loads. + DemoTable table; + for (int i = 0; i < 100; ++i) { + const auto id = table.insert(); + table.assign(id, i); + } + + const auto address = reinterpret_cast(table.column().data()); + EXPECT_EQ(address % soatable::simd_alignment, 0U); + EXPECT_GE(soatable::simd_alignment, 64U); +} + TEST(SpanTest, MakeRowIdRejectsDeadIndex) { DemoTable table; const auto id = table.insert(); From a15553e1f190b771347ff3b1b9f606fd7e88c3eb Mon Sep 17 00:00:00 2001 From: bbalouki Date: Wed, 24 Jun 2026 19:11:33 +0100 Subject: [PATCH 08/58] AoSoA tiled storage policy completes A.2 (FEATURES Phase A) Introduce a column storage policy so the table's layout is selectable without changing the row-facing API: - Refactor soa_table into basic_soa_table; soa_table is now an alias for the contiguous SoA default (non-breaking) and aosoa_table selects tiled storage. column_vector gains a Storage parameter. - Add soa_layout / aosoa_layout policies and detail::tiled_vector, a sequence of fixed-size, individually 64-byte-aligned tiles whose growth appends a tile and never copies existing tiles (bounded reallocation, stable element addresses). - column() is constrained to contiguous storage; column_tiles() gives a uniform per-tile aligned-span view across both policies. serialize.hpp handles both layouts (raw fast path for SoA, per-element for tiled). New aosoa tests (86 total). Verified: 86/86 ctest green; headers clean under clang -Wconversion -Wsign-conversion -Werror and MSVC /W4 /WX; AoSoA churn + sort + tile traversal + save/load ASan+UBSan clean. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 7 +- README.md | 1 + include/soatable/serialize.hpp | 41 +++--- include/soatable/soatable.hpp | 251 ++++++++++++++++++++++++++++----- tests/CMakeLists.txt | 1 + tests/unittests/aosoa_test.cpp | 146 +++++++++++++++++++ 6 files changed, 396 insertions(+), 51 deletions(-) create mode 100644 tests/unittests/aosoa_test.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index 676a22a..47ce9f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,12 @@ Foundational layout primitives (FEATURES Phase A). (A.4). - SIMD-aligned column storage: dense column data is over-aligned to `simd_alignment` (64 bytes, matching Apache Arrow) via an internal aligned allocator, so `column()` spans are ready for - aligned SIMD loads (A.2, alignment half). The optional AoSoA tiled layout remains pending. + aligned SIMD loads (A.2). +- Opt-in AoSoA tiled storage: `aosoa_table` stores each column's dense values + in fixed-size, individually over-aligned tiles (bounded reallocation; per-tile SIMD), selected via + a storage policy on the new `basic_soa_table`. `soa_table` is the unchanged + contiguous default. `column_tiles()` gives a uniform per-tile span view across both policies + (A.2). ## [0.3.0] - 2026-06-24 diff --git a/README.md b/README.md index 4d35ec5..ac6e53e 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ In our benchmarks, SoaTable's `select` queries are often **5x to 15x faster** th - **Zero-Copy Column Spans:** `column()` hands a column's dense values to BLAS / a SIMD kernel / numpy with no copy; `row_indices()` maps them back to handles. - **Validity Bitmaps:** `validity()` produces an Arrow-style packed presence bitmap for branchless masked iteration and interchange. - **Serialization:** opt-in `` `save()` / `load()` snapshot trivially-copyable tables to a versioned, schema-checked byte buffer. +- **Pluggable Storage Layout:** the default `soa_table` is flat and SIMD-aligned; `aosoa_table` opts into tiled (AoSoA-style) storage with bounded reallocation, and `column_tiles()` views either as per-tile aligned spans. ### Naming diff --git a/include/soatable/serialize.hpp b/include/soatable/serialize.hpp index 7264d8d..1e3ff2c 100644 --- a/include/soatable/serialize.hpp +++ b/include/soatable/serialize.hpp @@ -34,11 +34,12 @@ enum class serialize_status { namespace detail { -/// @brief Friend accessor exposing soa_table's private state to the serializer only. +/// @brief Friend accessor exposing a table's private state to the serializer only. +/// @tparam Storage The table's storage policy. /// @tparam Columns The table's column types. -template +template struct table_access { - using table_type = soa_table; + using table_type = basic_soa_table; static auto& rows(table_type& t) noexcept { return t.m_rows; } static const auto& rows(const table_type& t) noexcept { return t.m_rows; } @@ -128,18 +129,18 @@ struct byte_reader { /// @tparam Columns The table's column types. /// @param table The table to serialize. /// @return A byte buffer that load() can reconstruct the table from. -template -[[nodiscard]] std::vector save(const soa_table& table) { +template +[[nodiscard]] std::vector save(const basic_soa_table& table) { static_assert( (std::is_trivially_copyable_v && ...), "save() requires every column type to be trivially copyable." ); static_assert( - soa_table::column_count <= 64, + basic_soa_table::column_count <= 64, "save() supports up to 64 columns (the signature is stored as a 64-bit word)." ); - using access = detail::table_access; + using access = detail::table_access; std::vector out; const auto& rows = access::rows(table); @@ -164,16 +165,24 @@ template [&out](const auto&... pool) { ( [&out](const auto& column_pool) { - using value_type = std::decay_t; const auto& dense = column_pool.dense_rows(); const auto& data = column_pool.raw_data(); detail::append_pod(out, static_cast(dense.size())); detail::append_array( out, std::span(dense.data(), dense.size()) ); - detail::append_array( - out, std::span(data.data(), data.size()) - ); + using pool_type = std::decay_t; + if constexpr (pool_type::is_contiguous) { + using value_type = std::decay_t; + detail::append_array( + out, std::span(data.data(), data.size()) + ); + } else { + // Tiled storage is not contiguous; write each value individually. + for (std::size_t i = 0; i < data.size(); ++i) { + detail::append_pod(out, data[i]); + } + } }(pool), ...); }, @@ -188,20 +197,20 @@ template /// @param table The table to overwrite (cleared first). /// @param bytes The serialized buffer. /// @return serialize_status::ok on success, or the reason the buffer was rejected. -template +template [[nodiscard]] serialize_status load( - soa_table& table, std::span bytes + basic_soa_table& table, std::span bytes ) { static_assert( (std::is_trivially_copyable_v && ...), "load() requires every column type to be trivially copyable." ); static_assert( - soa_table::column_count <= 64, + basic_soa_table::column_count <= 64, "load() supports up to 64 columns (the signature is stored as a 64-bit word)." ); - using access = detail::table_access; + using access = detail::table_access; detail::byte_reader reader {bytes}; @@ -234,7 +243,7 @@ template for (auto& meta : rows) { meta.generation = reader.read_pod(); const auto sig = reader.read_pod(); - meta.signature = typename soa_table::signature_type {sig}; + meta.signature = typename basic_soa_table::signature_type {sig}; meta.alive = reader.read_pod() != 0; } diff --git a/include/soatable/soatable.hpp b/include/soatable/soatable.hpp index d052377..8e85b15 100644 --- a/include/soatable/soatable.hpp +++ b/include/soatable/soatable.hpp @@ -138,9 +138,10 @@ inline constexpr std::size_t column_alignment = namespace detail { /// @brief Friend accessor granting the opt-in header access to the private -/// state of a soa_table without widening its public surface. Defined only in that header. +/// state of a basic_soa_table without widening its public surface. Defined only in that header. +/// @tparam Storage The table's storage policy. /// @tparam Columns The table's column types. -template +template struct table_access; /// @brief A standard-conforming allocator that over-aligns its allocations to Alignment bytes. @@ -195,6 +196,96 @@ struct aligned_allocator { template using aligned_vector = std::vector>; +/// @brief A tiled (AoSoA-style) dense container: a sequence of fixed-size, individually over-aligned +/// tiles. Growth appends a tile and never copies existing tiles, bounding reallocation cost. +/// @tparam T The element type. +/// @tparam TileSize The number of elements per tile. +/// +/// Exposes the subset of the std::vector interface that column_vector relies on, so it is a drop-in +/// alternative storage backend. Elements are not contiguous across tiles, so it is non-contiguous. +template +class tiled_vector { + static_assert(TileSize > 0, "tiled_vector requires a positive tile size."); + + public: + /// @brief Number of stored elements. + [[nodiscard]] std::size_t size() const noexcept { return m_size; } + /// @brief Whether the container is empty. + [[nodiscard]] bool empty() const noexcept { return m_size == 0; } + + /// @brief Mutable element access by flat index. + [[nodiscard]] T& operator[](std::size_t index) { + return m_tiles[index / TileSize][index % TileSize]; + } + /// @brief Const element access by flat index. + [[nodiscard]] const T& operator[](std::size_t index) const { + return m_tiles[index / TileSize][index % TileSize]; + } + + /// @brief Construct an element at the end, allocating a new tile when the last one is full. + template + T& emplace_back(Args&&... args) { + if (m_tiles.empty() || m_tiles.back().size() == TileSize) { + m_tiles.emplace_back(); + m_tiles.back().reserve(TileSize); + } + T& ref = m_tiles.back().emplace_back(std::forward(args)...); + ++m_size; + return ref; + } + + /// @brief Append a value at the end. + void push_back(const T& value) { emplace_back(value); } + /// @brief Append a value at the end (move). + void push_back(T&& value) { emplace_back(std::move(value)); } + + /// @brief The last element. + [[nodiscard]] T& back() { return m_tiles.back().back(); } + + /// @brief Remove the last element, dropping the trailing tile when it becomes empty. + void pop_back() { + m_tiles.back().pop_back(); + if (m_tiles.back().empty()) { + m_tiles.pop_back(); + } + --m_size; + } + + /// @brief Reserve capacity for at least the given number of elements (rounded up to tiles). + void reserve(std::size_t capacity) { + m_tiles.reserve((capacity + TileSize - 1) / TileSize); + } + + /// @brief Remove all elements. + void clear() noexcept { + m_tiles.clear(); + m_size = 0; + } + + /// @brief Release unused capacity. + void shrink_to_fit() { + for (auto& tile_data : m_tiles) { + tile_data.shrink_to_fit(); + } + m_tiles.shrink_to_fit(); + } + + /// @brief Number of tiles currently allocated. + [[nodiscard]] std::size_t tile_count() const noexcept { return m_tiles.size(); } + /// @brief A contiguous, over-aligned span over one tile's live elements. + [[nodiscard]] std::span tile(std::size_t tile_index) { + return std::span(m_tiles[tile_index].data(), m_tiles[tile_index].size()); + } + /// @brief A contiguous, over-aligned const span over one tile's live elements. + [[nodiscard]] std::span tile(std::size_t tile_index) const { + return std::span(m_tiles[tile_index].data(), m_tiles[tile_index].size()); + } + + private: + std::vector> m_tiles; + std::size_t m_size = 0; +}; + /// @brief Generates a mask with the specified number of low bits set. /// @tparam UInt The unsigned integer type. /// @param bits The number of bits to set. @@ -267,6 +358,35 @@ using select_result_t = std::tuple< } // namespace detail +// ================================= +// 1b. Column storage policies +// ================================= + +/// @brief Storage policy: a flat, contiguous, over-aligned column (the default Structure-of-Arrays +/// layout). column() exposes a single contiguous span. +struct soa_layout { + /// @brief The dense container backing a column of T. + template + using container_type = detail::aligned_vector; + /// @brief Whether a column is a single contiguous block. + static constexpr bool is_contiguous = true; +}; + +/// @brief Storage policy: a tiled (AoSoA-style) column whose dense values live in fixed-size, +/// individually over-aligned tiles. Bounds reallocation cost on growth and is the natural unit for +/// chunked SIMD kernels; not contiguous, so use column_tiles() instead of column(). +/// @tparam TileSize The number of elements per tile. +template +struct aosoa_layout { + /// @brief The dense container backing a column of T. + template + using container_type = detail::tiled_vector; + /// @brief Tiled columns are not a single contiguous block. + static constexpr bool is_contiguous = false; + /// @brief The number of elements per tile. + static constexpr std::size_t tile_size = TileSize; +}; + // ================================= // 2. Compression and state helpers // ================================= @@ -544,7 +664,9 @@ class bitmap { /// Uses a sparse array (mapping row index to dense index) and a dense array /// (storing the actual data and the back-mapping to row index). /// This allows for O(1) insertion, deletion, and iteration over present values. -template +/// @tparam T The column value type. +/// @tparam Storage The storage policy selecting the dense backend (soa_layout or aosoa_layout). +template class column_vector { public: /// @brief Unsigned index type for dense and sparse positions. @@ -552,8 +674,10 @@ class column_vector { /// Widening this to `std::uint64_t` is the single change required for a 64-bit build; every /// internal index derives from it. Kept at 32 bits to match `row_id::index`. using size_type = std::uint32_t; - /// @brief The dense storage type: a vector over-aligned to column_alignment for SIMD loads. - using data_vector = detail::aligned_vector; + /// @brief The dense storage type chosen by the storage policy. + using data_vector = typename Storage::template container_type; + /// @brief Whether the dense storage is a single contiguous block. + static constexpr bool is_contiguous = Storage::is_contiguous; private: /// @brief Sentinel stored in m_sparse for a row that has no value in this column. @@ -809,9 +933,11 @@ struct row_view_element<0, Cols...> { /// - try_get() returns nullptr when the column is absent or the row is invalid. /// - get_expected() returns std::expected and never throws (for the no-exceptions audience). /// +/// @tparam Storage The column storage policy (soa_layout by default, or aosoa_layout). /// @tparam Columns The types of the columns in the table. Each type must be unique. -template -class soa_table { +/// @note Use the soa_table / aosoa_table aliases rather than naming basic_soa_table directly. +template +class basic_soa_table { public: static_assert(sizeof...(Columns) > 0, "soa_table must contain at least one column type."); static_assert( @@ -858,11 +984,11 @@ class soa_table { std::uint32_t m_free_head = npos; /// @brief Number of active rows. std::size_t m_alive_count = 0; - /// @brief Storage for each column. - std::tuple...> m_columns; + /// @brief Storage for each column, using the selected storage policy. + std::tuple...> m_columns; /// @brief Grants the opt-in serialization header access to the private state above. - friend struct detail::table_access; + friend struct detail::table_access; /// @brief Internal check for index overflow. [[nodiscard]] static constexpr bool can_address_rows(std::size_t count) { @@ -889,7 +1015,7 @@ class soa_table { const row_id id {row_index, self->m_rows[row_index].generation}; auto get_col = [&](std::uint32_t idx) { using ActualT = detail::optional_value_type_t; - auto* ptr = std::get>(self->m_columns).try_get(idx); + auto* ptr = std::get>(self->m_columns).try_get(idx); if constexpr (detail::is_optional_v) { using ResT = std::optional, const ActualT, ActualT>>>; @@ -907,7 +1033,7 @@ class soa_table { public: /// @brief Default constructor. - soa_table() = default; + basic_soa_table() = default; /// @brief Number of active rows in the table. [[nodiscard]] std::size_t size() const noexcept { return m_alive_count; } @@ -995,7 +1121,7 @@ class soa_table { const std::uint32_t index = id.index; - auto clear_row_from_pool = [index](column_vector& pool) { + auto clear_row_from_pool = [index](column_vector& pool) { if (pool.contains(index)) { pool.remove(index); } @@ -1034,7 +1160,7 @@ class soa_table { throw std::out_of_range("assign() called with an invalid row_id."); } - auto& pool = std::get>(m_columns); + auto& pool = std::get>(m_columns); T& value = pool.emplace(id.index, std::forward(args)...); m_rows[id.index].signature.set(index_of>::value); return value; @@ -1050,7 +1176,7 @@ class soa_table { return; } - auto& pool = std::get>(m_columns); + auto& pool = std::get>(m_columns); pool.remove(id.index); m_rows[id.index].signature.reset(index_of>::value); } @@ -1078,7 +1204,7 @@ class soa_table { if (!contains(id)) { return nullptr; } - return std::get>(m_columns).try_get(id.index); + return std::get>(m_columns).try_get(id.index); } /// @brief Try to get a const pointer to the value of a column for the given row. @@ -1091,7 +1217,7 @@ class soa_table { if (!contains(id)) { return nullptr; } - return std::get>(m_columns).try_get(id.index); + return std::get>(m_columns).try_get(id.index); } /// @brief Get a reference to the value of a column for the given row. Throws if not present. @@ -1136,7 +1262,7 @@ class soa_table { if (!is_valid(id)) { return std::unexpected(access_error::invalid_row); } - T* value = std::get>(m_columns).try_get(id.index); + T* value = std::get>(m_columns).try_get(id.index); if (value == nullptr) { return std::unexpected(access_error::missing_column); } @@ -1156,7 +1282,7 @@ class soa_table { if (!is_valid(id)) { return std::unexpected(access_error::invalid_row); } - const T* value = std::get>(m_columns).try_get(id.index); + const T* value = std::get>(m_columns).try_get(id.index); if (value == nullptr) { return std::unexpected(access_error::missing_column); } @@ -1217,7 +1343,7 @@ class soa_table { return m_rows[row_index].alive; }) | std::views::transform([this](std::size_t row_index) { - return get_transform_func(this)( + return get_transform_func(this)( static_cast(row_index) ); }); @@ -1228,7 +1354,7 @@ class soa_table { ( [&] { if constexpr (!detail::is_optional_v) { - auto& pool = std::get>(m_columns); + auto& pool = std::get>(m_columns); if (pool.size() < min_size) { min_size = pool.size(); driver_dense = &pool.dense_rows(); @@ -1245,7 +1371,7 @@ class soa_table { }; return *driver_dense | std::views::filter(std::move(filter_func)) | - std::views::transform(get_transform_func(this)); + std::views::transform(get_transform_func(this)); } } @@ -1278,7 +1404,7 @@ class soa_table { return m_rows[row_index].alive; }) | std::views::transform([this](std::size_t row_index) { - return get_transform_func(this)( + return get_transform_func(this)( static_cast(row_index) ); }); @@ -1289,7 +1415,7 @@ class soa_table { ( [&] { if constexpr (!detail::is_optional_v) { - auto& pool = std::get>(m_columns); + auto& pool = std::get>(m_columns); if (pool.size() < min_size) { min_size = pool.size(); driver_dense = &pool.dense_rows(); @@ -1306,7 +1432,8 @@ class soa_table { }; return *driver_dense | std::views::filter(std::move(filter_func)) | - std::views::transform(get_transform_func(this)); + std::views::transform(get_transform_func(this + )); } } @@ -1358,10 +1485,11 @@ class soa_table { /// @note The span covers only rows that currently have the column, in dense storage order; pair /// it with row_indices() to recover each value's row. Any operation that adds, removes, or /// reorders the column (assign, unassign, erase, sort, clear, reserve) invalidates the span. + /// Only available for contiguous (soa_layout) storage; tiled tables use column_tiles(). template - requires registered_column_v + requires registered_column_v && Storage::is_contiguous [[nodiscard]] std::span column() { - auto& data = std::get>(m_columns).raw_data(); + auto& data = std::get>(m_columns).raw_data(); return std::span(data.data(), data.size()); } @@ -1369,12 +1497,52 @@ class soa_table { /// @tparam T The column type. /// @return A span over the column's packed const values. template - requires registered_column_v + requires registered_column_v && Storage::is_contiguous [[nodiscard]] std::span column() const { - const auto& data = std::get>(m_columns).raw_data(); + const auto& data = std::get>(m_columns).raw_data(); return std::span(data.data(), data.size()); } + /// @brief View a column as a sequence of contiguous, over-aligned tiles. + /// @tparam T The column type. + /// @return One span per storage tile (a single whole-column span for soa_layout). + /// @note The uniform tiled view across both storage policies: feed each span to a SIMD kernel. + /// Invalidated by the same operations as column(). + template + requires registered_column_v + [[nodiscard]] std::vector> column_tiles() { + auto& data = std::get>(m_columns).raw_data(); + std::vector> tiles; + if constexpr (Storage::is_contiguous) { + tiles.emplace_back(data.data(), data.size()); + } else { + tiles.reserve(data.tile_count()); + for (std::size_t i = 0; i < data.tile_count(); ++i) { + tiles.push_back(data.tile(i)); + } + } + return tiles; + } + + /// @brief Const overload of column_tiles(). + /// @tparam T The column type. + /// @return One const span per storage tile. + template + requires registered_column_v + [[nodiscard]] std::vector> column_tiles() const { + const auto& data = std::get>(m_columns).raw_data(); + std::vector> tiles; + if constexpr (Storage::is_contiguous) { + tiles.emplace_back(data.data(), data.size()); + } else { + tiles.reserve(data.tile_count()); + for (std::size_t i = 0; i < data.tile_count(); ++i) { + tiles.push_back(data.tile(i)); + } + } + return tiles; + } + /// @brief Row indices parallel to column(), mapping each dense value back to its row. /// @tparam T The column type. /// @return A span where element i is the internal row index of column()[i]. @@ -1383,7 +1551,7 @@ class soa_table { template requires registered_column_v [[nodiscard]] std::span row_indices() const { - const auto& dense = std::get>(m_columns).dense_rows(); + const auto& dense = std::get>(m_columns).dense_rows(); return std::span(dense.data(), dense.size()); } @@ -1437,7 +1605,7 @@ class soa_table { template requires registered_column_v void assign_batch(const std::vector& ids, InputIt first) { - auto& pool = std::get>(m_columns); + auto& pool = std::get>(m_columns); auto it = first; for (auto id : ids) { if (is_valid(id)) { @@ -1472,7 +1640,7 @@ class soa_table { bool decided = (([&] { using T = typename std::decay_t::first_type; - auto& pool = std::get>(m_columns); + auto& pool = std::get>(m_columns); bool has_a = pool.contains(a); bool has_b = pool.contains(b); if (has_a && has_b) { @@ -1512,7 +1680,7 @@ class soa_table { template requires registered_column_v void sort_by_column(Compare&& comp) { - auto& pool = std::get>(m_columns); + auto& pool = std::get>(m_columns); const auto& dense = pool.dense_rows(); std::vector sorted_rows = dense; @@ -1559,7 +1727,7 @@ class soa_table { return; } - auto& pool = std::get>(m_columns); + auto& pool = std::get>(m_columns); const auto& dense = pool.dense_rows(); std::vector sorted_rows = dense; @@ -1612,6 +1780,21 @@ class soa_table { } }; +// ========================================== +// 4b. Table aliases by storage policy +// ========================================== + +/// @brief The default table: a flat, SIMD-aligned Structure-of-Arrays layout. +/// @tparam Columns The unique column types. +template +using soa_table = basic_soa_table; + +/// @brief A table whose columns use tiled (AoSoA-style) storage of the given tile size. +/// @tparam TileSize The number of elements per tile. +/// @tparam Columns The unique column types. +template +using aosoa_table = basic_soa_table, Columns...>; + // ===================== // 5. Row handle helper // ===================== diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 964b1a1..5e15718 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -41,6 +41,7 @@ set(SOATABLE_UNIT_TEST_SOURCES unittests/span_test.cpp unittests/validity_test.cpp unittests/serialize_test.cpp + unittests/aosoa_test.cpp unittests/const_correctness_test.cpp unittests/deprecated_alias_test.cpp unittests/include_hygiene_test.cpp diff --git a/tests/unittests/aosoa_test.cpp b/tests/unittests/aosoa_test.cpp new file mode 100644 index 0000000..e6a06ea --- /dev/null +++ b/tests/unittests/aosoa_test.cpp @@ -0,0 +1,146 @@ +// A.2 AoSoA tiled storage policy: aosoa_table behaves identically to soa_table for the row API, +// exposes per-tile aligned spans via column_tiles(), and tiles respect the configured size. + +#include + +#include +#include +#include + +#include "soatable/soatable.hpp" + +namespace { +struct Age { + int value = 0; +}; +struct Name { + int tag = 0; // Trivial stand-in so the type stays simple here. +}; + +constexpr std::size_t tile = 8; +using TiledTable = soatable::aosoa_table; +using FlatTable = soatable::soa_table; +} // namespace + +TEST(AosoaTest, RowApiMatchesSoa) { + TiledTable table; + const auto a = table.insert(); + table.assign(a, 30); + table.assign(a, 1); + const auto b = table.insert(); + table.assign(b, 20); + + EXPECT_EQ(table.get(a).value, 30); + EXPECT_TRUE(table.contains(a)); + EXPECT_FALSE(table.contains(b)); + + table.erase(a); + EXPECT_FALSE(table.is_valid(a)); + EXPECT_EQ(table.get(b).value, 20); +} + +TEST(AosoaTest, SelectAndSortWork) { + TiledTable table; + for (int i = 0; i < 20; ++i) { + const auto id = table.insert(); + table.assign(id, (20 - i) % 13); + table.assign(id, i); + } + table.sort_by_column([](const Age& l, const Age& r) { return l.value < r.value; }); + + int previous = -1; + int seen = 0; + for (auto [id, age, name] : table.select()) { + EXPECT_TRUE(table.is_valid(id)); + EXPECT_GE(age.get().value, previous); + previous = age.get().value; + static_cast(name); + ++seen; + } + EXPECT_EQ(seen, 20); +} + +TEST(AosoaTest, ColumnTilesCoverAllValuesInOrder) { + TiledTable table; + constexpr int row_count = 20; // More than two tiles of 8. + for (int i = 0; i < row_count; ++i) { + const auto id = table.insert(); + table.assign(id, i); + } + + std::vector flattened; + for (const std::span tile_span : table.column_tiles()) { + EXPECT_LE(tile_span.size(), tile); + EXPECT_EQ(reinterpret_cast(tile_span.data()) % soatable::simd_alignment, 0U); + for (const Age& age : tile_span) { + flattened.push_back(age.value); + } + } + + ASSERT_EQ(flattened.size(), static_cast(row_count)); + for (int i = 0; i < row_count; ++i) { + EXPECT_EQ(flattened[static_cast(i)], i); + } +} + +TEST(AosoaTest, FullTilesRespectTileSize) { + TiledTable table; + for (int i = 0; i < 17; ++i) { // 8 + 8 + 1 + const auto id = table.insert(); + table.assign(id, i); + } + + const auto tiles = table.column_tiles(); + ASSERT_EQ(tiles.size(), 3U); + EXPECT_EQ(tiles[0].size(), tile); + EXPECT_EQ(tiles[1].size(), tile); + EXPECT_EQ(tiles[2].size(), 1U); +} + +TEST(AosoaTest, MutationThroughTileIsVisible) { + TiledTable table; + const auto id = table.insert(); + table.assign(id, 5); + + auto tiles = table.column_tiles(); + ASSERT_FALSE(tiles.empty()); + tiles[0][0].value = 99; + + EXPECT_EQ(table.get(id).value, 99); +} + +TEST(AosoaTest, ProducesSameSelectionAsSoaUnderSameOps) { + TiledTable tiled; + FlatTable flat; + for (int i = 0; i < 50; ++i) { + const auto t = tiled.insert(); + const auto f = flat.insert(); + if (i % 3 != 0) { + tiled.assign(t, i); + flat.assign(f, i); + } + } + + std::vector from_tiled; + for (auto [id, age] : tiled.select()) { + static_cast(id); + from_tiled.push_back(age.get().value); + } + std::vector from_flat; + for (auto [id, age] : flat.select()) { + static_cast(id); + from_flat.push_back(age.get().value); + } + EXPECT_EQ(from_tiled, from_flat); +} + +TEST(AosoaTest, SoaColumnTilesReturnsSingleWholeColumn) { + FlatTable table; + for (int i = 0; i < 10; ++i) { + const auto id = table.insert(); + table.assign(id, i); + } + const auto tiles = table.column_tiles(); + ASSERT_EQ(tiles.size(), 1U); + EXPECT_EQ(tiles[0].size(), 10U); +} From 38ac58f4c01567beca5ad92983f2d95b262638c9 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Wed, 24 Jun 2026 19:20:57 +0100 Subject: [PATCH 09/58] Vectorized column ufuncs (FEATURES Phase B.1) Add opt-in , the start of the compute layer over the A.1 column spans: - Single-column span ufuncs (transform, reduce, inclusive_scan, count_if) that rely on compiler auto-vectorization over the 64-byte-aligned storage. - Table column helpers (transform_column, reduce_column, count_column_if) applied through column_tiles(), so they work uniformly for SoA and AoSoA storage. - Cross-column row-wise assign_from() that computes an output column from inputs over the select/view join (e.g. pnl = price * qty), in two passes so assigning the output never invalidates the in-progress join. New compute tests (91 total). Verified: 91/91 ctest green; clean under clang -Wconversion -Wsign-conversion -Werror and MSVC /W4 /WX; compute over AoSoA under churn ASan+UBSan clean. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 7 +- README.md | 1 + include/soatable/compute.hpp | 150 +++++++++++++++++++++++++++++++ tests/CMakeLists.txt | 1 + tests/unittests/compute_test.cpp | 110 +++++++++++++++++++++++ 5 files changed, 268 insertions(+), 1 deletion(-) create mode 100644 include/soatable/compute.hpp create mode 100644 tests/unittests/compute_test.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index 47ce9f4..2718c69 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -Foundational layout primitives (FEATURES Phase A). +Foundational layout primitives (FEATURES Phase A) and the start of the compute layer (Phase B). ### Added +- Opt-in `` vectorized column operations: single-column span ufuncs + (`transform`, `reduce`, `inclusive_scan`, `count_if`), policy-agnostic table helpers + (`transform_column`, `reduce_column`, `count_column_if`) over `column_tiles()`, and cross-column + row-wise `assign_from()` for expressions like `pnl = price * qty` (B.1). + - `column()` (and a `const` overload) returning a `std::span` over a column's dense values, plus `row_indices()` and `make_row_id()` to map dense positions back to stable handles (A.1). - `validity()` returning an Arrow-style packed `bitmap` of per-row column presence, with diff --git a/README.md b/README.md index ac6e53e..c1e57a5 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ In our benchmarks, SoaTable's `select` queries are often **5x to 15x faster** th - **Validity Bitmaps:** `validity()` produces an Arrow-style packed presence bitmap for branchless masked iteration and interchange. - **Serialization:** opt-in `` `save()` / `load()` snapshot trivially-copyable tables to a versioned, schema-checked byte buffer. - **Pluggable Storage Layout:** the default `soa_table` is flat and SIMD-aligned; `aosoa_table` opts into tiled (AoSoA-style) storage with bounded reallocation, and `column_tiles()` views either as per-tile aligned spans. +- **Vectorized Compute:** opt-in `` provides `transform` / `reduce` / `inclusive_scan` / `count_if` over column spans, policy-agnostic `transform_column` / `reduce_column`, and cross-column `assign_from()` (e.g. `pnl = price * qty`). ### Naming diff --git a/include/soatable/compute.hpp b/include/soatable/compute.hpp new file mode 100644 index 0000000..fd91b10 --- /dev/null +++ b/include/soatable/compute.hpp @@ -0,0 +1,150 @@ +#pragma once + +// Opt-in vectorized column operations ("the numpy part") layered on the zero-copy column spans from +// soatable.hpp. Single-column ufuncs operate on std::span and rely on compiler auto-vectorization +// over the 64-byte-aligned column storage; table-level helpers apply them through column_tiles(), +// so they work uniformly for the contiguous (SoA) and tiled (AoSoA) storage policies. Cross-column +// (row-wise) computation goes through the select/view join, since independent sparse columns are not +// row-aligned in storage. + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "soatable/soatable.hpp" + +namespace soatable::compute { + +// ========================= +// Single-column span ufuncs +// ========================= + +/// @brief Transform a column span in place. +/// @param values The span to modify. +/// @param op A callable mapping each element to its new value. +template +void transform(std::span values, UnaryOp op) { + std::transform(values.begin(), values.end(), values.begin(), op); +} + +/// @brief Transform an input span into an output span of equal length. +/// @param input The source span. +/// @param output The destination span (must be at least input.size()). +/// @param op A callable mapping an input element to an output element. +template +void transform(std::span input, std::span output, UnaryOp op) { + std::transform(input.begin(), input.end(), output.begin(), op); +} + +/// @brief Reduce a column span with an associative, commutative operation. +/// @param values The span to reduce. +/// @param init The identity / starting value. +/// @param op The reduction operation (defaults to addition). +/// @return The reduced value. +template > +[[nodiscard]] U reduce(std::span values, U init, BinaryOp op = {}) { + return std::reduce(values.begin(), values.end(), std::move(init), op); +} + +/// @brief Inclusive prefix scan of an input span into an output span. +/// @param input The source span. +/// @param output The destination span (must be at least input.size()). +/// @param op The combining operation (defaults to addition). +template > +void inclusive_scan(std::span input, std::span output, BinaryOp op = {}) { + std::inclusive_scan(input.begin(), input.end(), output.begin(), op); +} + +/// @brief Count the elements of a column span that satisfy a predicate. +/// @param values The span to scan. +/// @param pred The predicate. +/// @return The number of matching elements. +template +[[nodiscard]] std::size_t count_if(std::span values, Pred pred) { + return static_cast(std::count_if(values.begin(), values.end(), pred)); +} + +// ========================= +// Table column conveniences +// ========================= + +/// @brief Transform every value of a column in place (works for SoA and AoSoA storage). +/// @tparam T The column type. +/// @param table The table. +/// @param op A callable mapping each value to its new value. +template +void transform_column(basic_soa_table& table, UnaryOp op) { + for (std::span tile : table.template column_tiles()) { + compute::transform(tile, op); + } +} + +/// @brief Left-fold a column's values, allowing a projecting accumulator. +/// @tparam T The column type. +/// @param table The table. +/// @param init The starting accumulator value. +/// @param op A callable op(accumulator, const T&) returning the new accumulator. +/// @return The accumulated value. +template +[[nodiscard]] U reduce_column( + const basic_soa_table& table, U init, BinaryOp op +) { + U accumulator = std::move(init); + for (std::span tile : table.template column_tiles()) { + for (const T& value : tile) { + accumulator = op(std::move(accumulator), value); + } + } + return accumulator; +} + +/// @brief Count the rows whose value in column T satisfies a predicate. +/// @tparam T The column type. +/// @param table The table. +/// @param pred The predicate over const T&. +/// @return The number of matching values. +template +[[nodiscard]] std::size_t count_column_if( + const basic_soa_table& table, Pred pred +) { + std::size_t total = 0; + for (std::span tile : table.template column_tiles()) { + total += compute::count_if(tile, pred); + } + return total; +} + +// ========================= +// Cross-column (row-wise) computation +// ========================= + +/// @brief Compute an output column from one or more input columns, per row. +/// +/// Iterates the rows that have every input column (the select/view join) and assigns the result of +/// @p op to the output column. Runs in two passes so assigning the output never invalidates the +/// in-progress join. +/// @tparam Out The output column type (must differ from every input). +/// @tparam In The input column types. +/// @param table The table. +/// @param op A callable op(const In&...) returning an Out value. +template +void assign_from(basic_soa_table& table, Op op) { + static_assert( + (!std::same_as && ...), "assign_from: the output column must differ from inputs." + ); + + std::vector> results; + for (auto row : table.template view()) { + results.emplace_back(row.id(), op(row.template get()...)); + } + for (auto& [id, value] : results) { + table.template assign(id, std::move(value)); + } +} + +} // namespace soatable::compute diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 5e15718..cee4f27 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -42,6 +42,7 @@ set(SOATABLE_UNIT_TEST_SOURCES unittests/validity_test.cpp unittests/serialize_test.cpp unittests/aosoa_test.cpp + unittests/compute_test.cpp unittests/const_correctness_test.cpp unittests/deprecated_alias_test.cpp unittests/include_hygiene_test.cpp diff --git a/tests/unittests/compute_test.cpp b/tests/unittests/compute_test.cpp new file mode 100644 index 0000000..b779b27 --- /dev/null +++ b/tests/unittests/compute_test.cpp @@ -0,0 +1,110 @@ +// B.1 vectorized column ops: single-column span ufuncs, table column helpers across SoA and AoSoA +// storage, and cross-column row-wise assign_from. + +#include + +#include +#include + +#include "soatable/compute.hpp" +#include "soatable/soatable.hpp" + +namespace { +struct Price { + double value = 0.0; +}; +struct Qty { + int value = 0; +}; +struct Pnl { + double value = 0.0; +}; +using Book = soatable::soa_table; +using TiledBook = soatable::aosoa_table<4, Price, Qty, Pnl>; +} // namespace + +TEST(ComputeTest, SpanTransformReduceScanCount) { + std::vector data = {1, 2, 3, 4}; + soatable::compute::transform(std::span(data), [](int x) { return x * 10; }); + EXPECT_EQ(data, (std::vector {10, 20, 30, 40})); + + const int sum = soatable::compute::reduce(std::span(data), 0); + EXPECT_EQ(sum, 100); + + std::vector scanned(data.size()); + soatable::compute::inclusive_scan(std::span(data), std::span(scanned)); + EXPECT_EQ(scanned, (std::vector {10, 30, 60, 100})); + + EXPECT_EQ(soatable::compute::count_if(std::span(data), [](int x) { return x >= 30; }), 2U); +} + +TEST(ComputeTest, TransformColumnInPlaceSoa) { + Book book; + for (int i = 0; i < 5; ++i) { + const auto id = book.insert(); + book.assign(id, Price {static_cast(i)}); + } + soatable::compute::transform_column(book, [](Price p) { return Price {p.value + 1.0}; }); + + const double total = + soatable::compute::reduce_column(book, 0.0, [](double acc, const Price& p) { + return acc + p.value; + }); + EXPECT_DOUBLE_EQ(total, 1.0 + 2.0 + 3.0 + 4.0 + 5.0); +} + +TEST(ComputeTest, ColumnOpsWorkOnTiledStorage) { + TiledBook book; + for (int i = 0; i < 10; ++i) { // Spans multiple tiles of 4. + const auto id = book.insert(); + book.assign(id, Qty {i}); + } + soatable::compute::transform_column(book, [](Qty q) { return Qty {q.value * 2}; }); + + const int sum = soatable::compute::reduce_column(book, 0, [](int acc, const Qty& q) { + return acc + q.value; + }); + EXPECT_EQ(sum, 2 * (0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9)); + + const auto big = soatable::compute::count_column_if(book, [](const Qty& q) { + return q.value >= 10; + }); + EXPECT_EQ(big, 5U); // doubled values 10,12,14,16,18 +} + +TEST(ComputeTest, AssignFromComputesCrossColumn) { + Book book; + for (int i = 1; i <= 4; ++i) { + const auto id = book.insert(); + book.assign(id, Price {static_cast(i)}); + book.assign(id, Qty {i}); + } + + soatable::compute::assign_from(book, [](const Price& p, const Qty& q) { + return Pnl {p.value * q.value}; + }); + + double sum_pnl = 0.0; + for (auto [id, pnl] : book.select()) { + static_cast(id); + sum_pnl += pnl.get().value; + } + EXPECT_DOUBLE_EQ(sum_pnl, 1.0 + 4.0 + 9.0 + 16.0); +} + +TEST(ComputeTest, AssignFromOnlyTouchesRowsWithAllInputs) { + Book book; + const auto both = book.insert(); + book.assign(both, Price {2.0}); + book.assign(both, Qty {3}); + const auto price_only = book.insert(); + book.assign(price_only, Price {5.0}); // No Qty: excluded. + + soatable::compute::assign_from(book, [](const Price& p, const Qty& q) { + return Pnl {p.value * q.value}; + }); + + EXPECT_TRUE(book.contains(both)); + EXPECT_DOUBLE_EQ(book.get(both).value, 6.0); + EXPECT_FALSE(book.contains(price_only)); +} From ed8139d88ba4b533593be47d9c36815fd29112a7 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Wed, 24 Jun 2026 19:29:02 +0100 Subject: [PATCH 10/58] Query DSL: predicate filtering and group-by aggregation (FEATURES Phase B.2) Add opt-in : - select_where(table, pred): lazily composes view() with a row predicate (the smallest-driver scan plus a filter), yielding row_view proxies. - group_reduce(table, key_of, init, op) and the conveniences group_sum / group_count: group rows by a projected key and fold a value column, building analytics aggregates on top of select<>(). New query tests (96 total). Verified: 96/96 ctest green; clean under clang -Wconversion -Wsign-conversion -Werror and MSVC /W4 /WX; query under churn ASan+UBSan clean. Co-Authored-By: Claude Opus 4.8 --- .gitignore | 1 + CHANGELOG.md | 2 + README.md | 1 + benchmarks/CMakeLists.txt | 2 +- cmake/Versions.cmake | 1 + include/soatable/query.hpp | 97 +++++++++++++++++++++++++++++++ tests/CMakeLists.txt | 3 +- tests/unittests/query_test.cpp | 101 +++++++++++++++++++++++++++++++++ 8 files changed, 206 insertions(+), 2 deletions(-) create mode 100644 include/soatable/query.hpp create mode 100644 tests/unittests/query_test.cpp diff --git a/.gitignore b/.gitignore index 9d2bab5..1ab3fa0 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ prompts.txt # Generated artifacts dist/ bench/ +dist/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 2718c69..84b62ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,8 @@ Foundational layout primitives (FEATURES Phase A) and the start of the compute l (`transform`, `reduce`, `inclusive_scan`, `count_if`), policy-agnostic table helpers (`transform_column`, `reduce_column`, `count_column_if`) over `column_tiles()`, and cross-column row-wise `assign_from()` for expressions like `pnl = price * qty` (B.1). +- Opt-in `` query helpers: `select_where()` predicate filtering over the + smallest-driver scan, and group-by aggregation (`group_reduce` / `group_sum` / `group_count`) (B.2). - `column()` (and a `const` overload) returning a `std::span` over a column's dense values, plus `row_indices()` and `make_row_id()` to map dense positions back to stable handles (A.1). diff --git a/README.md b/README.md index c1e57a5..fb8a5f7 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,7 @@ In our benchmarks, SoaTable's `select` queries are often **5x to 15x faster** th - **Serialization:** opt-in `` `save()` / `load()` snapshot trivially-copyable tables to a versioned, schema-checked byte buffer. - **Pluggable Storage Layout:** the default `soa_table` is flat and SIMD-aligned; `aosoa_table` opts into tiled (AoSoA-style) storage with bounded reallocation, and `column_tiles()` views either as per-tile aligned spans. - **Vectorized Compute:** opt-in `` provides `transform` / `reduce` / `inclusive_scan` / `count_if` over column spans, policy-agnostic `transform_column` / `reduce_column`, and cross-column `assign_from()` (e.g. `pnl = price * qty`). +- **Query Helpers:** opt-in `` adds `select_where()` predicate filtering and group-by aggregation (`group_sum` / `group_count` / `group_reduce`). ### Naming diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 8ddfaa3..8c8b88b 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -5,7 +5,7 @@ if(NOT benchmark_FOUND) FetchContent_Declare( benchmark GIT_REPOSITORY https://github.com/google/benchmark.git - GIT_TAG v1.9.5 + GIT_TAG "v${BENCHMARCK_VERSION}" ) set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "" FORCE) set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "" FORCE) diff --git a/cmake/Versions.cmake b/cmake/Versions.cmake index aa73787..8a2eef5 100644 --- a/cmake/Versions.cmake +++ b/cmake/Versions.cmake @@ -1,3 +1,4 @@ cmake_minimum_required(VERSION 3.20) set(GTEST_VERSION 1.17.0 CACHE STRING "") +set(BENCHMARCK_VERSION 1.9.5 CACHE STRING "") diff --git a/include/soatable/query.hpp b/include/soatable/query.hpp new file mode 100644 index 0000000..0dfd4ec --- /dev/null +++ b/include/soatable/query.hpp @@ -0,0 +1,97 @@ +#pragma once + +// Opt-in query helpers layered on select<>() / view<>(): predicate filtering and group-by +// aggregation. These compose the existing smallest-driver scan with a predicate or a grouping pass, +// giving analytics ergonomics without leaving C++. + +#include +#include +#include +#include +#include + +#include "soatable/soatable.hpp" + +namespace soatable::query { + +/// @brief Select rows that have all of Cols and additionally satisfy a row predicate. +/// @tparam Cols The required column types (non-optional). +/// @param table The table to query. +/// @param predicate A callable taking a row_view and returning bool. +/// @return A range of row_view proxies for the matching rows (structured-binding friendly). +/// @note Lazy: it composes view() with std::views::filter. The table must outlive iteration. +template +[[nodiscard]] auto select_where(basic_soa_table& table, Pred predicate) { + return table.template view() | + std::views::filter([predicate](auto row) { return predicate(row); }); +} + +/// @brief Group rows by a key derived from column Key and fold column Value within each group. +/// @tparam Key The key column type. +/// @tparam Value The value column type. +/// @param table The table to aggregate. +/// @param key_of A callable mapping const Key& to a hashable, equality-comparable group key. +/// @param init The per-group starting accumulator value. +/// @param op A callable op(accumulator, const Value&) returning the new accumulator. +/// @return A map from group key to accumulated value, over rows that have both Key and Value. +template < + typename Key, typename Value, typename Storage, typename... Columns, typename KeyProj, typename U, + typename Op> +[[nodiscard]] auto group_reduce( + const basic_soa_table& table, KeyProj key_of, U init, Op op +) { + using key_type = std::decay_t>; + std::unordered_map groups; + for (auto [id, key, value] : table.template select()) { + static_cast(id); + const key_type group_key = key_of(key.get()); + auto it = groups.find(group_key); + if (it == groups.end()) { + it = groups.emplace(group_key, init).first; + } + it->second = op(std::move(it->second), value.get()); + } + return groups; +} + +/// @brief Group rows by a key and sum a projection of column Value within each group. +/// @tparam Key The key column type. +/// @tparam Value The value column type. +/// @param table The table to aggregate. +/// @param key_of A callable mapping const Key& to a hashable group key. +/// @param value_of A callable mapping const Value& to a summable number. +/// @return A map from group key to the summed projection. +template < + typename Key, typename Value, typename Storage, typename... Columns, typename KeyProj, + typename ValueProj> +[[nodiscard]] auto group_sum( + const basic_soa_table& table, KeyProj key_of, ValueProj value_of +) { + using sum_type = std::decay_t>; + return group_reduce( + table, key_of, sum_type {}, + [value_of](sum_type accumulator, const Value& value) { + return accumulator + value_of(value); + } + ); +} + +/// @brief Count the rows in each group keyed by a projection of column Key. +/// @tparam Key The key column type. +/// @param table The table to aggregate. +/// @param key_of A callable mapping const Key& to a hashable group key. +/// @return A map from group key to the number of rows that have Key in that group. +template +[[nodiscard]] auto group_count( + const basic_soa_table& table, KeyProj key_of +) { + using key_type = std::decay_t>; + std::unordered_map counts; + for (auto [id, key] : table.template select()) { + static_cast(id); + ++counts[key_of(key.get())]; + } + return counts; +} + +} // namespace soatable::query diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index cee4f27..e208b10 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -5,7 +5,7 @@ if(NOT GTest_FOUND) FetchContent_Declare( GTest GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG v1.17.0 + GIT_TAG "v${GTEST_VERSION}" ) set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) FetchContent_MakeAvailable(GTest) @@ -43,6 +43,7 @@ set(SOATABLE_UNIT_TEST_SOURCES unittests/serialize_test.cpp unittests/aosoa_test.cpp unittests/compute_test.cpp + unittests/query_test.cpp unittests/const_correctness_test.cpp unittests/deprecated_alias_test.cpp unittests/include_hygiene_test.cpp diff --git a/tests/unittests/query_test.cpp b/tests/unittests/query_test.cpp new file mode 100644 index 0000000..febe67e --- /dev/null +++ b/tests/unittests/query_test.cpp @@ -0,0 +1,101 @@ +// B.2 query DSL: select_where predicate filtering and group_by aggregation (group_reduce/sum/count). + +#include + +#include +#include + +#include "soatable/query.hpp" +#include "soatable/soatable.hpp" + +namespace { +struct Region { + int id = 0; +}; +struct Pnl { + double value = 0.0; +}; +struct Active { + bool on = false; +}; +using Trades = soatable::soa_table; + +Trades make_trades() { + Trades table; + for (int i = 0; i < 9; ++i) { + const auto id = table.insert(); + table.assign(id, Region {i % 3}); + table.assign(id, Pnl {static_cast(i)}); + if (i % 2 == 0) { + table.assign(id, Active {true}); + } + } + return table; +} +} // namespace + +TEST(QueryTest, SelectWhereFiltersRows) { + Trades table = make_trades(); + int matches = 0; + for (auto row : soatable::query::select_where(table, [](auto r) { + return r.template get().value >= 5.0; + })) { + EXPECT_GE(row.template get().value, 5.0); + ++matches; + } + EXPECT_EQ(matches, 4); // pnl 5,6,7,8 +} + +TEST(QueryTest, SelectWhereYieldsMutableReferences) { + Trades table = make_trades(); + for (auto row : soatable::query::select_where(table, [](auto r) { + return r.template get().value == 0.0; + })) { + row.template get().value = 100.0; + } + + int updated = 0; + for (auto [id, pnl] : table.select()) { + static_cast(id); + if (pnl.get().value == 100.0) { + ++updated; + } + } + EXPECT_EQ(updated, 1); +} + +TEST(QueryTest, GroupSumByRegion) { + Trades table = make_trades(); + const auto sums = soatable::query::group_sum( + table, [](const Region& r) { return r.id; }, [](const Pnl& p) { return p.value; } + ); + + // Region 0: rows 0,3,6 -> 9; Region 1: 1,4,7 -> 12; Region 2: 2,5,8 -> 15. + ASSERT_EQ(sums.size(), 3U); + EXPECT_DOUBLE_EQ(sums.at(0), 9.0); + EXPECT_DOUBLE_EQ(sums.at(1), 12.0); + EXPECT_DOUBLE_EQ(sums.at(2), 15.0); +} + +TEST(QueryTest, GroupCountByRegion) { + Trades table = make_trades(); + const auto counts = soatable::query::group_count(table, [](const Region& r) { + return r.id; + }); + ASSERT_EQ(counts.size(), 3U); + EXPECT_EQ(counts.at(0), 3U); + EXPECT_EQ(counts.at(1), 3U); + EXPECT_EQ(counts.at(2), 3U); +} + +TEST(QueryTest, GroupReduceComputesPerGroupMax) { + Trades table = make_trades(); + const auto maxima = + soatable::query::group_reduce( + table, [](const Region& r) { return r.id; }, 0.0, + [](double acc, const Pnl& p) { return std::max(acc, p.value); } + ); + EXPECT_DOUBLE_EQ(maxima.at(0), 6.0); + EXPECT_DOUBLE_EQ(maxima.at(1), 7.0); + EXPECT_DOUBLE_EQ(maxima.at(2), 8.0); +} From 39d68cf7b839f147f44b837c070f6979249b1260 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Wed, 24 Jun 2026 19:40:05 +0100 Subject: [PATCH 11/58] Broadcasting, masked, and strided column ops (FEATURES Phase B.3) Extend with the subset/broadcast adapters that feed the B.1 ufuncs: - broadcast / add_scalar / multiply_scalar: apply a scalar across a column span. - transform_if (value-flagged), transform_masked (parallel byte mask), and transform_strided (every n-th element) for operating on a subset of a column. - Table-level broadcast_column and transform_column_if, applied through column_tiles() so they work for both SoA and AoSoA storage. This completes FEATURES Phase B (compute layer). New tests (102 total). Verified: 102/102 ctest green; clean under clang -Wconversion -Wsign-conversion -Werror and MSVC /W4 /WX; B.3 ops over AoSoA under churn ASan+UBSan clean. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 3 ++ include/soatable/compute.hpp | 92 ++++++++++++++++++++++++++++++++ tests/unittests/compute_test.cpp | 65 ++++++++++++++++++++++ 3 files changed, 160 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 84b62ee..e7dcf7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,9 @@ Foundational layout primitives (FEATURES Phase A) and the start of the compute l row-wise `assign_from()` for expressions like `pnl = price * qty` (B.1). - Opt-in `` query helpers: `select_where()` predicate filtering over the smallest-driver scan, and group-by aggregation (`group_reduce` / `group_sum` / `group_count`) (B.2). +- Broadcasting, masked, and strided column ops in ``: `broadcast` / + `add_scalar` / `multiply_scalar`, `transform_if` / `transform_masked` / `transform_strided`, and the + table-level `broadcast_column` / `transform_column_if` (B.3). - `column()` (and a `const` overload) returning a `std::span` over a column's dense values, plus `row_indices()` and `make_row_id()` to map dense positions back to stable handles (A.1). diff --git a/include/soatable/compute.hpp b/include/soatable/compute.hpp index fd91b10..d970205 100644 --- a/include/soatable/compute.hpp +++ b/include/soatable/compute.hpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -119,6 +120,97 @@ template return total; } +// ========================= +// Broadcasting, masked, and strided ops +// ========================= + +/// @brief Broadcast a scalar over a column span in place: values[i] = op(values[i], scalar). +/// @param values The span to modify. +/// @param scalar The scalar operand applied to every element. +/// @param op The combining operation (defaults to addition). +template > +void broadcast(std::span values, const Scalar& scalar, BinaryOp op = {}) { + for (T& value : values) { + value = op(value, scalar); + } +} + +/// @brief Add a scalar to every element of a column span (a column bias). +template +void add_scalar(std::span values, const Scalar& scalar) { + broadcast(values, scalar, std::plus<> {}); +} + +/// @brief Multiply every element of a column span by a scalar. +template +void multiply_scalar(std::span values, const Scalar& scalar) { + broadcast(values, scalar, std::multiplies<> {}); +} + +/// @brief Transform only the elements that satisfy a predicate (a value-flagged subset). +/// @param values The span to modify. +/// @param pred A predicate over const T& selecting which elements to transform. +/// @param op A callable mapping a selected element to its new value. +template +void transform_if(std::span values, Pred pred, UnaryOp op) { + for (T& value : values) { + if (pred(value)) { + value = op(value); + } + } +} + +/// @brief Transform the elements flagged by a parallel byte mask of equal length. +/// @param values The span to modify. +/// @param mask A span where a non-zero entry flags the element at the same index. +/// @param op A callable mapping a flagged element to its new value. +template +void transform_masked(std::span values, std::span mask, UnaryOp op) { + const std::size_t count = std::min(values.size(), mask.size()); + for (std::size_t i = 0; i < count; ++i) { + if (mask[i] != 0) { + values[i] = op(values[i]); + } + } +} + +/// @brief Transform every stride-th element of a column span (a strided subset). +/// @param values The span to modify. +/// @param stride The step between transformed elements (must be positive). +/// @param op A callable mapping a selected element to its new value. +template +void transform_strided(std::span values, std::size_t stride, UnaryOp op) { + for (std::size_t i = 0; i < values.size(); i += stride) { + values[i] = op(values[i]); + } +} + +/// @brief Broadcast a scalar over a whole column in place (works for SoA and AoSoA storage). +/// @tparam T The column type. +/// @param table The table. +/// @param scalar The scalar operand. +/// @param op The combining operation (defaults to addition). +template > +void broadcast_column(basic_soa_table& table, const Scalar& scalar, + BinaryOp op = {}) { + for (std::span tile : table.template column_tiles()) { + compute::broadcast(tile, scalar, op); + } +} + +/// @brief Transform a column's flagged values in place across both storage policies. +/// @tparam T The column type. +/// @param table The table. +/// @param pred A predicate over const T& selecting which values to transform. +/// @param op A callable mapping a selected value to its new value. +template +void transform_column_if(basic_soa_table& table, Pred pred, UnaryOp op) { + for (std::span tile : table.template column_tiles()) { + compute::transform_if(tile, pred, op); + } +} + // ========================= // Cross-column (row-wise) computation // ========================= diff --git a/tests/unittests/compute_test.cpp b/tests/unittests/compute_test.cpp index b779b27..646a6ca 100644 --- a/tests/unittests/compute_test.cpp +++ b/tests/unittests/compute_test.cpp @@ -3,6 +3,7 @@ #include +#include #include #include @@ -92,6 +93,70 @@ TEST(ComputeTest, AssignFromComputesCrossColumn) { EXPECT_DOUBLE_EQ(sum_pnl, 1.0 + 4.0 + 9.0 + 16.0); } +TEST(ComputeTest, BroadcastAddAndMultiplyScalar) { + std::vector data = {1.0, 2.0, 3.0}; + soatable::compute::add_scalar(std::span(data), 10.0); + EXPECT_EQ(data, (std::vector {11.0, 12.0, 13.0})); + soatable::compute::multiply_scalar(std::span(data), 2.0); + EXPECT_EQ(data, (std::vector {22.0, 24.0, 26.0})); +} + +TEST(ComputeTest, TransformIfOnlyFlaggedElements) { + std::vector data = {1, 6, 3, 8, 4}; + soatable::compute::transform_if( + std::span(data), [](int x) { return x > 4; }, [](int x) { return x * 10; } + ); + EXPECT_EQ(data, (std::vector {1, 60, 3, 80, 4})); +} + +TEST(ComputeTest, TransformMaskedByParallelMask) { + std::vector data = {1, 2, 3, 4, 5}; + std::vector mask = {1, 0, 1, 0, 1}; + soatable::compute::transform_masked( + std::span(data), std::span(mask), [](int x) { return -x; } + ); + EXPECT_EQ(data, (std::vector {-1, 2, -3, 4, -5})); +} + +TEST(ComputeTest, TransformStridedEveryNth) { + std::vector data = {1, 1, 1, 1, 1, 1}; + soatable::compute::transform_strided(std::span(data), 2, [](int x) { return x + 9; }); + EXPECT_EQ(data, (std::vector {10, 1, 10, 1, 10, 1})); +} + +TEST(ComputeTest, BroadcastColumnBiasAcrossTiles) { + TiledBook book; + for (int i = 0; i < 10; ++i) { + const auto id = book.insert(); + book.assign(id, Price {static_cast(i)}); + } + soatable::compute::broadcast_column(book, 100.0, [](Price p, double s) { + return Price {p.value + s}; + }); + + const double total = + soatable::compute::reduce_column(book, 0.0, [](double acc, const Price& p) { + return acc + p.value; + }); + EXPECT_DOUBLE_EQ(total, (0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9) + 10 * 100.0); +} + +TEST(ComputeTest, TransformColumnIfAcrossTiles) { + TiledBook book; + for (int i = 0; i < 10; ++i) { + const auto id = book.insert(); + book.assign(id, Qty {i}); + } + soatable::compute::transform_column_if( + book, [](const Qty& q) { return q.value % 2 == 0; }, [](Qty q) { return Qty {q.value + 100}; } + ); + + const auto biased = soatable::compute::count_column_if(book, [](const Qty& q) { + return q.value >= 100; + }); + EXPECT_EQ(biased, 5U); // even values 0,2,4,6,8 +} + TEST(ComputeTest, AssignFromOnlyTouchesRowsWithAllInputs) { Book book; const auto both = book.insert(); From f30cb54143d9542b3e8a9edaccc6624a8d6246c3 Mon Sep 17 00:00:00 2001 From: bbalouki Date: Wed, 24 Jun 2026 19:54:21 +0100 Subject: [PATCH 12/58] Allocator-aware columns + std::pmr support (FEATURES Phase C.2) - Add the soa_layout_with storage policy and the custom_soa_table< Allocator, Columns...> alias, so column storage can use any caller-supplied allocator template. The default soa_layout now derives from soa_layout_with, keeping the 64-byte over-alignment and the existing API unchanged. - Add opt-in with pmr_allocator and pmr_soa_table, routing column allocations through the active std::pmr memory resource (arena / monotonic buffer / pool) for deterministic, fragmentation-free behaviour. New allocator tests (105 total). Verified: 105/105 ctest green; clean under clang -Wconversion -Wsign-conversion -Werror and MSVC /W4 /WX. Co-Authored-By: Claude Opus 4.8 --- CHANGELOG.md | 3 + CMakeLists.txt | 4 +- README.md | 1 + VERSION.txt | 1 + include/soatable/pmr.hpp | 27 +++++++++ include/soatable/soatable.hpp | 30 ++++++++-- tests/CMakeLists.txt | 1 + tests/unittests/allocator_test.cpp | 92 ++++++++++++++++++++++++++++++ 8 files changed, 152 insertions(+), 7 deletions(-) create mode 100644 VERSION.txt create mode 100644 include/soatable/pmr.hpp create mode 100644 tests/unittests/allocator_test.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index e7dcf7a..2c39404 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,9 @@ Foundational layout primitives (FEATURES Phase A) and the start of the compute l - Broadcasting, masked, and strided column ops in ``: `broadcast` / `add_scalar` / `multiply_scalar`, `transform_if` / `transform_masked` / `transform_strided`, and the table-level `broadcast_column` / `transform_column_if` (B.3). +- Allocator-aware columns: `soa_layout_with` storage policy and `custom_soa_table` let columns use a caller-supplied allocator; opt-in `` adds + `pmr_soa_table` backed by the active `std::pmr` memory resource (C.2). - `column()` (and a `const` overload) returning a `std::span` over a column's dense values, plus `row_indices()` and `make_row_id()` to map dense positions back to stable handles (A.1). diff --git a/CMakeLists.txt b/CMakeLists.txt index 45fdde3..ead02b5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,8 @@ cmake_minimum_required(VERSION 3.25) -project(SOATABLE VERSION 0.3.0 LANGUAGES CXX) +file(READ "VERSION.txt" RAW_VERSION) +string(STRIP "${RAW_VERSION}" APP_VERSION) +project(SOATABLE VERSION ${APP_VERSION} LANGUAGES CXX) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) diff --git a/README.md b/README.md index fb8a5f7..d517529 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,7 @@ In our benchmarks, SoaTable's `select` queries are often **5x to 15x faster** th - **Pluggable Storage Layout:** the default `soa_table` is flat and SIMD-aligned; `aosoa_table` opts into tiled (AoSoA-style) storage with bounded reallocation, and `column_tiles()` views either as per-tile aligned spans. - **Vectorized Compute:** opt-in `` provides `transform` / `reduce` / `inclusive_scan` / `count_if` over column spans, policy-agnostic `transform_column` / `reduce_column`, and cross-column `assign_from()` (e.g. `pnl = price * qty`). - **Query Helpers:** opt-in `` adds `select_where()` predicate filtering and group-by aggregation (`group_sum` / `group_count` / `group_reduce`). +- **Custom Allocators:** `custom_soa_table` plugs in any allocator; opt-in `` `pmr_soa_table` routes column storage through a `std::pmr` arena / monotonic / pool resource. ### Naming diff --git a/VERSION.txt b/VERSION.txt new file mode 100644 index 0000000..0d91a54 --- /dev/null +++ b/VERSION.txt @@ -0,0 +1 @@ +0.3.0 diff --git a/include/soatable/pmr.hpp b/include/soatable/pmr.hpp new file mode 100644 index 0000000..e5413ef --- /dev/null +++ b/include/soatable/pmr.hpp @@ -0,0 +1,27 @@ +#pragma once + +// Opt-in std::pmr (polymorphic memory resource) support for SoaTable. Included separately so the +// core header does not pull in . A pmr_soa_table allocates its column storage from +// the active std::pmr default memory resource, letting callers route allocations to an arena, +// monotonic buffer, or pool resource for deterministic, fragmentation-free behaviour. + +#include + +#include "soatable/soatable.hpp" + +namespace soatable { + +/// @brief A single-argument allocator template wrapping std::pmr::polymorphic_allocator. +/// @tparam T The element type. +template +using pmr_allocator = std::pmr::polymorphic_allocator; + +/// @brief A flat table whose columns allocate from the active std::pmr default memory resource. +/// @tparam Columns The unique column types. +/// +/// Set the resource with std::pmr::set_default_resource() (or wrap construction accordingly) before +/// populating the table to route its allocations through an arena / monotonic buffer / pool. +template +using pmr_soa_table = custom_soa_table; + +} // namespace soatable diff --git a/include/soatable/soatable.hpp b/include/soatable/soatable.hpp index 8e85b15..a6d7fd3 100644 --- a/include/soatable/soatable.hpp +++ b/include/soatable/soatable.hpp @@ -362,16 +362,28 @@ using select_result_t = std::tuple< // 1b. Column storage policies // ================================= -/// @brief Storage policy: a flat, contiguous, over-aligned column (the default Structure-of-Arrays -/// layout). column() exposes a single contiguous span. -struct soa_layout { - /// @brief The dense container backing a column of T. +/// @brief Storage policy: a flat, contiguous column allocated with a caller-chosen allocator. +/// @tparam Allocator A single-argument allocator template applied per element type +/// (e.g. std::pmr::polymorphic_allocator). Compose with custom_soa_table. +template